Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

hotfix: Tests for layoutParsing

Browse Source
This commit is contained in:
deiflaender 2023-05-12 12:29:13 +02:00
parent e4a034700a
commit 2ae601f08e
14 changed files with 321 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java
View File

@ -4,8 +4,8 @@ import static java.util.stream.Collectors.toSet;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.regex.Matcher;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.StringFreq
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil; import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling;
@ -43,19 +44,34 @@ public class BlockificationService {
float minX = 1000, maxX = 0, minY = 1000, maxY = 0; float minX = 1000, maxX = 0, minY = 1000, maxY = 0;
TextPositionSequence prev = null; TextPositionSequence prev = null;
var pattern = Patterns.getCompiledPattern("\\b(?:[1-9]|1\\d|20|[ivxlc]+)\\s*(?:[.)])", true);
var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false);
boolean wasSplitted = false; boolean wasSplitted = false;
Float splitX1 = null; Float splitX1 = null;
for (TextPositionSequence word : textPositions) { for (TextPositionSequence word : textPositions) {
Matcher matcher = pattern.matcher(word.toString());
Matcher matcher2 = pattern2.matcher(word.toString());
Matcher matcher3 = pattern3.matcher(word.toString());
boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25; boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25;
boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight(); boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight();
boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj(); boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj();
boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX; boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() - minX < -5;
boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj(); boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj();
boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines); boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines);
boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir()); boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir());
boolean splitOnFontDifferent = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle().equals(prev.getFontStyle()) || word.getFontSize() !=prev.getFontSize());
boolean newline = prev != null && Math.abs(word.getMinYDirAdj()- prev.getMinYDirAdj()) > word.getHeight();
boolean isNumericalIdentifier = matcher.matches();
if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) { if (prev != null && (prev.isParagraphStart() || xIsBeforeFirstX || splitByX || lineSeparation || startFromTop || isSplitByRuling || (newline && splitOnFontDifferent) || (isNumericalIdentifier && newline))) {
// if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) {
Orientation prevOrientation = null; Orientation prevOrientation = null;
if (!chunkBlockList1.isEmpty()) { if (!chunkBlockList1.isEmpty()) {
@ -107,51 +123,51 @@ public class BlockificationService {
chunkBlockList1.add(cb1); chunkBlockList1.add(cb1);
} }
Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator(); // Iterator<AbstractTextContainer> itty = chunkBlockList1.iterator();
TextBlock previousLeft = null; // TextBlock previousLeft = null;
TextBlock previousRight = null; // TextBlock previousRight = null;
while (itty.hasNext()) { // while (itty.hasNext()) {
TextBlock block = (TextBlock) itty.next(); // TextBlock block = (TextBlock) itty.next();
//
if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { // if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) {
if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) { // if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) {
previousLeft.add(block); // previousLeft.add(block);
itty.remove(); // itty.remove();
continue; // continue;
} // }
} // }
//
if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) { // if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) {
if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) { // if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) {
previousRight.add(block); // previousRight.add(block);
itty.remove(); // itty.remove();
continue; // continue;
} // }
} // }
//
if (block.getOrientation().equals(Orientation.LEFT)) { // if (block.getOrientation().equals(Orientation.LEFT)) {
previousLeft = block; // previousLeft = block;
} else if (block.getOrientation().equals(Orientation.RIGHT)) { // } else if (block.getOrientation().equals(Orientation.RIGHT)) {
previousRight = block; // previousRight = block;
} // }
} // }
//
itty = chunkBlockList1.iterator(); // itty = chunkBlockList1.iterator();
TextBlock previous = null; // TextBlock previous = null;
while (itty.hasNext()) { // while (itty.hasNext()) {
TextBlock block = (TextBlock) itty.next(); // TextBlock block = (TextBlock) itty.next();
//
if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), // if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(),
previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() // previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation()
.equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) { // .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) {
previous.add(block); // previous.add(block);
itty.remove(); // itty.remove();
continue; // continue;
} // }
//
previous = block; // previous = block;
} // }
return new Page(chunkBlockList1); return new Page(chunkBlockList1);
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java
View File

@ -84,7 +84,7 @@ public class BodyTextFrameService {
} }
float approxLineCount = PositionUtils.getApproxLineCount(textBlock); float approxLineCount = PositionUtils.getApproxLineCount(textBlock);
if (approxLineCount < 3.3f) { if (approxLineCount < 2.9f) {
continue; continue;
} }

117
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java
View File

@ -52,64 +52,67 @@ public class ClassificationService {
var bodyTextFrame = page.getBodyTextFrame(); var bodyTextFrame = page.getBodyTextFrame();
if (document.getFontSizeCounter().getMostPopular() == null) {
textBlock.setClassification("Other");
return;
}
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) {
textBlock.setClassification("Header");
} else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
textBlock.setClassification("Footer");
} else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
.size() == 1)) {
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
textBlock.setClassification("Title");
}
} else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
.getCountPerValue()
.containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
.get(0)
.getTextPositions()
.get(0)
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
for (int i = 1; i <= headlineFontSizes.size(); i++) {
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
textBlock.setClassification("H " + i);
document.setHeadlines(true);
}
}
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
.getMostPopular()
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
.get(0)
.getTextPositions()
.get(0)
.getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
document.setHeadlines(true);
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
textBlock.setClassification("TextBlock Bold");
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
.equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
textBlock.setClassification("TextBlock"); textBlock.setClassification("TextBlock");
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() // return;
.getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
.getMostPopular() // if (document.getFontSizeCounter().getMostPopular() == null) {
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { // textBlock.setClassification("Other");
textBlock.setClassification("TextBlock Italic"); // return;
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) { // }
textBlock.setClassification("TextBlock Unknown"); // if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) {
} else { // textBlock.setClassification("Header");
textBlock.setClassification("Other"); //
} // } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter()
// .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) {
// textBlock.setClassification("Footer");
// } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock,
// document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks()
// .size() == 1)) {
// if (!Pattern.matches("[0-9]+", textBlock.toString())) {
// textBlock.setClassification("Title");
// }
// } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter()
// .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter()
// .getCountPerValue()
// .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences()
// .get(0)
// .getTextPositions()
// .get(0)
// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
//
// for (int i = 1; i <= headlineFontSizes.size(); i++) {
// if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
// textBlock.setClassification("H " + i);
// document.setHeadlines(true);
// }
// }
// } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame,
// textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
// .getMostPopular()
// .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences()
// .get(0)
// .getTextPositions()
// .get(0)
// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
// textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
// document.setHeadlines(true);
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
// .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
// textBlock.setClassification("TextBlock Bold");
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
// .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
// .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) {
// textBlock.setClassification("TextBlock");
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
// .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter()
// .getMostPopular()
// .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
// textBlock.setClassification("TextBlock Italic");
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
// textBlock.setClassification("TextBlock Unknown");
// } else {
// textBlock.setClassification("Other");
// }
} }
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java
View File

@ -91,6 +91,9 @@ public class RedactionController implements RedactionResource {
} else { } else {
var table = (Table) t; var table = (Table) t;
if(table == null){
continue;
}
for (var row: table.getRows()){ for (var row: table.getRows()){

82
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java
View File

@ -1,82 +0,0 @@
package com.iqser.red.service.redaction.v1.server.parsing;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import lombok.Getter;
import lombok.Setter;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import org.apache.pdfbox.text.TextPosition;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class PDFAreaTextStripper extends PDFTextStripperByArea {
@Getter
private List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@Setter
private int pageNumber;
public PDFAreaTextStripper() throws IOException {
}
@Override
public void writeString(String text, List<TextPosition> textPositions) throws IOException {
int startIndex = 0;
for (int i = 0; i <= textPositions.size() - 1; i++) {
if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) {
startIndex++;
continue;
}
// Strange but sometimes this is happening, for example: Metolachlor2.pdf
if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) {
List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
}
startIndex = i;
}
if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) {
List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
}
startIndex = i;
}
if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) {
List<TextPosition> sublist = textPositions.subList(startIndex, i);
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
}
startIndex = i + 1;
}
}
List<TextPosition> sublist = textPositions.subList(startIndex, textPositions.size());
if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) {
sublist = sublist.subList(0, sublist.size() - 1);
}
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber));
}
super.writeString(text);
}
public void clearPositions() {
textPositionSequences = new ArrayList<>();
}
}

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java
View File

@ -195,7 +195,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
@Override @Override
public void writeString(String text, List<TextPosition> textPositions) throws IOException { public void writeString(String text, List<TextPosition> textPositions, boolean isParagraphStart) throws IOException {
int startIndex = 0; int startIndex = 0;
RedTextPosition previous = null; RedTextPosition previous = null;
@ -235,7 +235,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
.getUnicode() .getUnicode()
.equals("\t")))) { .equals("\t")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
} }
startIndex = i; startIndex = i;
} }
@ -245,7 +245,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0) if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0)
.getUnicode() .getUnicode()
.equals("\t")))) { .equals("\t")))) {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
} }
startIndex = i; startIndex = i;
} }
@ -265,7 +265,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
textPositionSequences.get(textPositionSequences.size() - 1).add(t); textPositionSequences.get(textPositionSequences.size() - 1).add(t);
} }
} else { } else {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart));
} }
} }
startIndex = i + 1; startIndex = i + 1;
@ -288,7 +288,7 @@ public class PDFLinesTextStripper extends PDFTextStripper {
textPositionSequences.get(textPositionSequences.size() - 1).add(t); textPositionSequences.get(textPositionSequences.size() - 1).add(t);
} }
} else { } else {
textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart));
} }
} }
super.writeString(text); super.writeString(text);

59
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java
View File

@ -27,6 +27,7 @@ import java.text.Bidi;
import java.text.Normalizer; import java.text.Normalizer;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
@ -240,10 +241,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
document = doc; document = doc;
output = outputStream; output = outputStream;
if (getAddMoreFormatting()) { if (getAddMoreFormatting()) {
paragraphEnd = lineSeparator; paragraphEnd = "\n----ParagraphEnd----\n\n";
pageStart = lineSeparator; pageStart = lineSeparator;
articleStart = lineSeparator; articleStart = "\n----ArticelStart----\n\n";
articleEnd = lineSeparator; articleEnd = "\n----ArticelEnd----\n\n";
} }
startDocument(document); startDocument(document);
processPages(document.getPages()); processPages(document.getPages());
@ -594,9 +595,14 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
// but this caused a lot of regression test failures. So, I'm leaving it be for // but this caused a lot of regression test failures. So, I'm leaving it be for
// now // now
if (!overlap(positionY, positionHeight, maxYForLine, maxHeightForLine)) { if (!overlap(positionY, positionHeight, maxYForLine, maxHeightForLine)) {
writeLine(normalize(line)); var normalized = normalize(line);
line.clear(); // normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent()
lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine); lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine);
writeLine(normalized, current.isParagraphStart);
line.clear();
expectedStartOfNextWordX = EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE; expectedStartOfNextWordX = EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE;
maxYForLine = MAX_Y_FOR_LINE_RESET_VALUE; maxYForLine = MAX_Y_FOR_LINE_RESET_VALUE;
maxHeightForLine = MAX_HEIGHT_FOR_LINE_RESET_VALUE; maxHeightForLine = MAX_HEIGHT_FOR_LINE_RESET_VALUE;
@ -630,7 +636,24 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
if (startOfPage && lastPosition == null) { if (startOfPage && lastPosition == null) {
writeParagraphStart();// not sure this is correct for RTL? writeParagraphStart();// not sure this is correct for RTL?
} }
line.add(new LineItem(position)); line.add(new LineItem(position));
// Collections.sort(line, new Comparator<LineItem>() {
//
// @Override
// public int compare(LineItem str1, LineItem str2) {
// if(null == str1.getTextPosition()) {
// return 0;
// }
// else if(null == str2.getTextPosition()) {
// return 0;
// }
// return Float.compare(str1.getTextPosition().getX(), str2.getTextPosition().getX());
// }
// });
// line.sort(Comparator.comparing(a -> a.getTextPosition() != null && a.getTextPosition().getX()));
} }
maxHeightForLine = Math.max(maxHeightForLine, positionHeight); maxHeightForLine = Math.max(maxHeightForLine, positionHeight);
minYTopForLine = Math.min(minYTopForLine, positionY - positionHeight); minYTopForLine = Math.min(minYTopForLine, positionY - positionHeight);
@ -646,7 +669,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
} }
// print the final line // print the final line
if (line.size() > 0) { if (line.size() > 0) {
writeLine(normalize(line)); writeLine(normalize(line), false);
writeParagraphEnd(); writeParagraphEnd();
} }
endArticle(); endArticle();
@ -703,7 +726,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
* @param textPositions The TextPositions belonging to the text. * @param textPositions The TextPositions belonging to the text.
* @throws IOException If there is an error when writing the text. * @throws IOException If there is an error when writing the text.
*/ */
protected void writeString(String text, List<TextPosition> textPositions) throws IOException { protected void writeString(String text, List<TextPosition> textPositions, boolean isParagraphEnd) throws IOException {
writeString(text); writeString(text);
} }
@ -1385,6 +1408,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
} else { } else {
writeLineSeparator(); writeLineSeparator();
writeParagraphSeparator(); writeParagraphSeparator();
lastLineStartPosition.setEndParagraphWritten();
} }
} else { } else {
writeLineSeparator(); writeLineSeparator();
@ -1428,6 +1452,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
float newXVal = multiplyFloat(getIndentThreshold(), position.getTextPosition().getWidthOfSpace()); float newXVal = multiplyFloat(getIndentThreshold(), position.getTextPosition().getWidthOfSpace());
float positionWidth = multiplyFloat(0.25f, position.getTextPosition().getWidth()); float positionWidth = multiplyFloat(0.25f, position.getTextPosition().getWidth());
// if(xGap < 0){
// result = true;
// }
// else
if (yGap > newYVal) { if (yGap > newYVal) {
result = true; result = true;
} else if (xGap > newXVal) { } else if (xGap > newXVal) {
@ -1636,12 +1664,13 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
* @param line a list with the words of the given line * @param line a list with the words of the given line
* @throws IOException if something went wrong * @throws IOException if something went wrong
*/ */
private void writeLine(List<WordWithTextPositions> line) throws IOException { private void writeLine(List<WordWithTextPositions> line, boolean isParagraphEnd) throws IOException {
int numberOfStrings = line.size(); int numberOfStrings = line.size();
for (int i = 0; i < numberOfStrings; i++) { for (int i = 0; i < numberOfStrings; i++) {
WordWithTextPositions word = line.get(i); WordWithTextPositions word = line.get(i);
writeString(word.getText(), word.getTextPositions()); word.getTextPositions().sort(Comparator.comparing(TextPosition::getX));
writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1);
if (i < numberOfStrings - 1) { if (i < numberOfStrings - 1) {
writeWordSeparator(); writeWordSeparator();
} }
@ -1963,6 +1992,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
private boolean isHangingIndent = false; private boolean isHangingIndent = false;
private boolean isArticleStart = false; private boolean isArticleStart = false;
private boolean endParagraphWritten = false;
private TextPosition position = null; private TextPosition position = null;
@ -2024,6 +2055,16 @@ public class PDFTextStripper extends LegacyPDFStreamEngine {
} }
public boolean isEndParagraphWritten() {
return endParagraphWritten;
}
public void setEndParagraphWritten(){
endParagraphWritten = true;
}
/** /**
* Sets the isArticleStart() flag to true. * Sets the isArticleStart() flag to true.
*/ */

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java
View File

@ -39,6 +39,7 @@ public class TextPositionSequence implements CharSequence {
private int rotation; private int rotation;
private float pageHeight; private float pageHeight;
private float pageWidth; private float pageWidth;
private boolean isParagraphStart;
public TextPositionSequence(int page) { public TextPositionSequence(int page) {
@ -47,7 +48,7 @@ public class TextPositionSequence implements CharSequence {
} }
public TextPositionSequence(List<TextPosition> textPositions, int page) { public TextPositionSequence(List<TextPosition> textPositions, int page, boolean isParagraphStart) {
this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList()); this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList());
this.page = page; this.page = page;
@ -55,6 +56,7 @@ public class TextPositionSequence implements CharSequence {
this.rotation = textPositions.get(0).getRotation(); this.rotation = textPositions.get(0).getRotation();
this.pageHeight = textPositions.get(0).getPageHeight(); this.pageHeight = textPositions.get(0).getPageHeight();
this.pageWidth = textPositions.get(0).getPageWidth(); this.pageWidth = textPositions.get(0).getPageWidth();
this.isParagraphStart = isParagraphStart;
} }

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -10,6 +10,7 @@ import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.PosixFilePermissions;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
@ -20,6 +21,7 @@ import org.apache.pdfbox.io.MemoryUsageSetting;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.common.PDRectangle; import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.util.QuickSort;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@ -96,7 +98,18 @@ public class PdfSegmentationService {
stripper.setStartPage(pageNumber); stripper.setStartPage(pageNumber);
stripper.setEndPage(pageNumber); stripper.setEndPage(pageNumber);
stripper.setPdpage(pdPage); stripper.setPdpage(pdPage);
stripper.getText(pdDocument); stripper.setAddMoreFormatting(true);
stripper.setShouldSeparateByBeads(true);
// stripper.setDropThreshold(0.1f);
var text = stripper.getText(pdDocument);
System.out.println(text);
// stripper.getTextPositionSequences().forEach(a -> {
// System.out.println(a);
// if(a.isParagraphEnd()){
// System.out.println("------------------------");
// }
// });
PDRectangle pdr = pdPage.getMediaBox(); PDRectangle pdr = pdPage.getMediaBox();
@ -110,13 +123,35 @@ public class PdfSegmentationService {
stripper.getMaxCharHeight()); stripper.getMaxCharHeight());
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical());
// QuickSort.sort(page.getTextBlocks(), new TextBlockComparator());
//
// List<AbstractTextContainer> combined = new ArrayList<>();
// Iterator<AbstractTextContainer> itty = page.getTextBlocks().iterator();
// TextBlock prev = null;
// while (itty.hasNext()){
// TextBlock current = (TextBlock) itty.next();
//
// if(prev != null && (((Math.abs(prev.getMaxX() - current.getMaxX()) < 1)|| Math.abs(prev.getMinX() - current.getMinX()) < 1) && current.getMaxY() > prev.getMaxY() && current.getMinY() - prev.getMaxY() < 3)){
// combined.remove(prev);
// current = prev.union(current);
// }
//
// combined.add(current);
//
// prev= current;
// }
//
// page.setTextBlocks(combined);
page.setRotation(rotation); page.setRotation(rotation);
page.setLandscape(isLandscape); page.setLandscape(isLandscape);
page.setPageNumber(pageNumber); page.setPageNumber(pageNumber);
page.setPageWidth(cropbox.getWidth()); page.setPageWidth(cropbox.getWidth());
page.setPageHeight(cropbox.getHeight()); page.setPageHeight(cropbox.getHeight());
tableExtractionService.extractTables(cleanRulings, page); // tableExtractionService.extractTables(cleanRulings, page);
buildPageStatistics(page); buildPageStatistics(page);
increaseDocumentStatistics(page, document); increaseDocumentStatistics(page, document);

70
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TextBlockComparator.java Normal file
View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.iqser.red.service.redaction.v1.server.segmentation;
import java.util.Comparator;
import org.apache.pdfbox.text.TextPosition;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
/**
* This class is a comparator for TextPosition operators. It handles
* pages with text in different directions by grouping the text based
* on direction and sorting in that direction. This allows continuous text
* in a given direction to be more easily grouped together.
*
* @author Ben Litchfield
*/
public class TextBlockComparator implements Comparator<AbstractTextContainer>
{
@Override
public int compare(AbstractTextContainer pos1, AbstractTextContainer pos2)
{
// get the text direction adjusted coordinates
float x1 = pos1.getMinX();
float x2 = pos2.getMinX();
float pos1YBottom = pos1.getMaxY();
float pos2YBottom = pos2.getMaxY();
// note that the coordinates have been adjusted so 0,0 is in upper left
float pos1YTop = pos1YBottom - pos1.getHeight();
float pos2YTop = pos2YBottom - pos2.getHeight();
float yDifference = Math.abs(pos1YBottom - pos2YBottom);
// we will do a simple tolerance comparison
if (yDifference < .1 ||
pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom ||
pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom)
{
return Float.compare(x1, x2);
}
else if (pos1YBottom < pos2YBottom)
{
return -1;
}
else
{
return 1;
}
}
}

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java
View File

@ -82,14 +82,14 @@ public class PdfVisualisationService {
} }
} }
contentStream.setStrokingColor(Color.YELLOW); // contentStream.setStrokingColor(Color.YELLOW);
contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), // contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(),
analyzedPage.getBodyTextFrame().getTopLeft().getY(), // analyzedPage.getBodyTextFrame().getTopLeft().getY(),
analyzedPage.getBodyTextFrame().getWidth(), // analyzedPage.getBodyTextFrame().getWidth(),
analyzedPage.getBodyTextFrame().getHeight()); // analyzedPage.getBodyTextFrame().getHeight());
//
contentStream.stroke(); // contentStream.stroke();
//
contentStream.close(); contentStream.close();
} }
} }
@ -103,15 +103,15 @@ public class PdfVisualisationService {
contentStream.stroke(); contentStream.stroke();
if (textBlock.getClassification() != null) { if (textBlock.getClassification() != null) {
contentStream.beginText(); // contentStream.beginText();
//
contentStream.setNonStrokingColor(Color.BLUE); // contentStream.setNonStrokingColor(Color.BLUE);
contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f); // contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f);
//
contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2); // contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2);
contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir()); // contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir());
//
contentStream.endText(); // contentStream.endText();
if (DRAW_POSITIONS) { if (DRAW_POSITIONS) {
drawPositions(contentStream, textBlock); drawPositions(contentStream, textBlock);

16
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1117,8 +1117,8 @@ public class RedactionIntegrationTest {
System.out.println("classificationTest"); System.out.println("classificationTest");
String sourcePath ="/tmp"; String sourcePath ="/tmp/Input";
String targetPath = "/tmp/result"; String targetPath = "/tmp/Output";
var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath); var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath);
@ -1179,7 +1179,7 @@ public class RedactionIntegrationTest {
System.out.println("classificationTest"); System.out.println("classificationTest");
String tableServiceResponseFile = "files/cv_table_response_VV-511309.json"; String tableServiceResponseFile = "files/cv_table_response_VV-511309.json";
AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile); AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf");
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.dossierId(request.getDossierId()) .dossierId(request.getDossierId())
@ -1220,7 +1220,7 @@ public class RedactionIntegrationTest {
public void htmlTablesTest() throws IOException { public void htmlTablesTest() throws IOException {
System.out.println("htmlTablesTest"); System.out.println("htmlTablesTest");
AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf"); AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf");
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.dossierId(request.getDossierId()) .dossierId(request.getDossierId())
@ -1241,7 +1241,7 @@ public class RedactionIntegrationTest {
System.out.println("htmlTableRotationTest"); System.out.println("htmlTableRotationTest");
AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf");
RedactionRequest redactionRequest = RedactionRequest.builder() RedactionRequest redactionRequest = RedactionRequest.builder()
.dossierId(request.getDossierId()) .dossierId(request.getDossierId())
@ -1254,6 +1254,12 @@ public class RedactionIntegrationTest {
try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) { try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) {
fileOutputStream.write(result.getDocument()); fileOutputStream.write(result.getDocument());
} }
} }

0
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table.pdf → redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf
View File

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/S8Plenar.pdf Normal file
View File

Binary file not shown.