diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java index 2cff711a..5335afbf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BlockificationService.java @@ -4,8 +4,8 @@ import static java.util.stream.Collectors.toSet; import java.util.ArrayList; import java.util.Comparator; -import java.util.Iterator; import java.util.List; +import java.util.regex.Matcher; import org.springframework.stereotype.Service; @@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.StringFreq import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.utils.RulingTextDirAdjustUtil; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.Ruling; @@ -43,19 +44,34 @@ public class BlockificationService { float minX = 1000, maxX = 0, minY = 1000, maxY = 0; TextPositionSequence prev = null; + + var pattern = Patterns.getCompiledPattern("\\b(?:[1-9]|1\\d|20|[ivxlc]+)\\s*(?:[.)])", true); + var pattern2 = Patterns.getCompiledPattern(".*\\d$", true); + var pattern3 = Patterns.getCompiledPattern("^(\\d{1,1}\\.){1,3}\\d{1,2}\\.?\\s[a-z]{1,2}\\/[a-z]{1,2}.*", false); + + + boolean wasSplitted = false; Float splitX1 = null; for (TextPositionSequence word : textPositions) { + Matcher matcher = pattern.matcher(word.toString()); + Matcher matcher2 = pattern2.matcher(word.toString()); + Matcher matcher3 = pattern3.matcher(word.toString()); + boolean lineSeparation = word.getMinYDirAdj() - maxY > word.getHeight() * 1.25; boolean startFromTop = prev != null && word.getMinYDirAdj() < prev.getMinYDirAdj() - prev.getTextHeight(); boolean splitByX = prev != null && maxX + 50 < word.getMinXDirAdj() && prev.getMinYDirAdj() == word.getMinYDirAdj(); - boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() < minX; + boolean xIsBeforeFirstX = prev != null && word.getMinXDirAdj() - minX < -5; boolean newLineAfterSplit = prev != null && word.getMinYDirAdj() != prev.getMinYDirAdj() && wasSplitted && splitX1 != word.getMinXDirAdj(); boolean isSplitByRuling = isSplitByRuling(minX, minY, maxX, maxY, word, horizontalRulingLines, verticalRulingLines); boolean splitByDir = prev != null && !prev.getDir().equals(word.getDir()); + boolean splitOnFontDifferent = prev != null && (!word.getFont().equals(prev.getFont()) || !word.getFontStyle().equals(prev.getFontStyle()) || word.getFontSize() !=prev.getFontSize()); + boolean newline = prev != null && Math.abs(word.getMinYDirAdj()- prev.getMinYDirAdj()) > word.getHeight(); + boolean isNumericalIdentifier = matcher.matches(); - if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) { + if (prev != null && (prev.isParagraphStart() || xIsBeforeFirstX || splitByX || lineSeparation || startFromTop || isSplitByRuling || (newline && splitOnFontDifferent) || (isNumericalIdentifier && newline))) { +// if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) { Orientation prevOrientation = null; if (!chunkBlockList1.isEmpty()) { @@ -107,51 +123,51 @@ public class BlockificationService { chunkBlockList1.add(cb1); } - Iterator itty = chunkBlockList1.iterator(); +// Iterator itty = chunkBlockList1.iterator(); - TextBlock previousLeft = null; - TextBlock previousRight = null; - while (itty.hasNext()) { - TextBlock block = (TextBlock) itty.next(); - - if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { - if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) { - previousLeft.add(block); - itty.remove(); - continue; - } - } - - if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) { - if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) { - previousRight.add(block); - itty.remove(); - continue; - } - } - - if (block.getOrientation().equals(Orientation.LEFT)) { - previousLeft = block; - } else if (block.getOrientation().equals(Orientation.RIGHT)) { - previousRight = block; - } - } - - itty = chunkBlockList1.iterator(); - TextBlock previous = null; - while (itty.hasNext()) { - TextBlock block = (TextBlock) itty.next(); - - if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), - previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() - .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) { - previous.add(block); - itty.remove(); - continue; - } - - previous = block; - } +// TextBlock previousLeft = null; +// TextBlock previousRight = null; +// while (itty.hasNext()) { +// TextBlock block = (TextBlock) itty.next(); +// +// if (previousLeft != null && block.getOrientation().equals(Orientation.LEFT)) { +// if (previousLeft.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousLeft.getMinY()) { +// previousLeft.add(block); +// itty.remove(); +// continue; +// } +// } +// +// if (previousRight != null && block.getOrientation().equals(Orientation.RIGHT)) { +// if (previousRight.getMinY() > block.getMinY() && block.getMaxY() + block.getMostPopularWordHeight() > previousRight.getMinY()) { +// previousRight.add(block); +// itty.remove(); +// continue; +// } +// } +// +// if (block.getOrientation().equals(Orientation.LEFT)) { +// previousLeft = block; +// } else if (block.getOrientation().equals(Orientation.RIGHT)) { +// previousRight = block; +// } +// } +// +// itty = chunkBlockList1.iterator(); +// TextBlock previous = null; +// while (itty.hasNext()) { +// TextBlock block = (TextBlock) itty.next(); +// +// if (previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation().equals(Orientation.LEFT) && equalsWithThreshold(block.getMaxY(), +// previous.getMaxY()) || previous != null && previous.getOrientation().equals(Orientation.LEFT) && block.getOrientation() +// .equals(Orientation.RIGHT) && equalsWithThreshold(block.getMaxY(), previous.getMaxY())) { +// previous.add(block); +// itty.remove(); +// continue; +// } +// +// previous = block; +// } return new Page(chunkBlockList1); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java index 1dc54ae2..809e9120 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/BodyTextFrameService.java @@ -84,7 +84,7 @@ public class BodyTextFrameService { } float approxLineCount = PositionUtils.getApproxLineCount(textBlock); - if (approxLineCount < 3.3f) { + if (approxLineCount < 2.9f) { continue; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java index a7773af8..1d310b22 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/service/ClassificationService.java @@ -52,64 +52,67 @@ public class ClassificationService { var bodyTextFrame = page.getBodyTextFrame(); - if (document.getFontSizeCounter().getMostPopular() == null) { - textBlock.setClassification("Other"); - return; - } - if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) { - textBlock.setClassification("Header"); + textBlock.setClassification("TextBlock"); +// return; - } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() - .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { - textBlock.setClassification("Footer"); - } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, - document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks() - .size() == 1)) { - if (!Pattern.matches("[0-9]+", textBlock.toString())) { - textBlock.setClassification("Title"); - } - } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() - .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() - .getCountPerValue() - .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences() - .get(0) - .getTextPositions() - .get(0) - .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { - - for (int i = 1; i <= headlineFontSizes.size(); i++) { - if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { - textBlock.setClassification("H " + i); - document.setHeadlines(true); - } - } - } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, - textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() - .getMostPopular() - .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() - .get(0) - .getTextPositions() - .get(0) - .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { - textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); - document.setHeadlines(true); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() - .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { - textBlock.setClassification("TextBlock Bold"); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() - .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle() - .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) { - textBlock.setClassification("TextBlock"); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() - .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter() - .getMostPopular() - .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { - textBlock.setClassification("TextBlock Italic"); - } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) { - textBlock.setClassification("TextBlock Unknown"); - } else { - textBlock.setClassification("Other"); - } +// if (document.getFontSizeCounter().getMostPopular() == null) { +// textBlock.setClassification("Other"); +// return; +// } +// if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) ) { +// textBlock.setClassification("Header"); +// +// } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() +// .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { +// textBlock.setClassification("Footer"); +// } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, +// document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks() +// .size() == 1)) { +// if (!Pattern.matches("[0-9]+", textBlock.toString())) { +// textBlock.setClassification("Title"); +// } +// } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() +// .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() +// .getCountPerValue() +// .containsKey("bold") && textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter().getMostPopular() + 1) && textBlock.getSequences() +// .get(0) +// .getTextPositions() +// .get(0) +// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { +// +// for (int i = 1; i <= headlineFontSizes.size(); i++) { +// if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { +// textBlock.setClassification("H " + i); +// document.setHeadlines(true); +// } +// } +// } else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, +// textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() +// .getMostPopular() +// .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() +// .get(0) +// .getTextPositions() +// .get(0) +// .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { +// textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); +// document.setHeadlines(true); +// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() +// .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { +// textBlock.setClassification("TextBlock Bold"); +// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() +// .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle() +// .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) { +// textBlock.setClassification("TextBlock"); +// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() +// .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter() +// .getMostPopular() +// .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { +// textBlock.setClassification("TextBlock Italic"); +// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) { +// textBlock.setClassification("TextBlock Unknown"); +// } else { +// textBlock.setClassification("Other"); +// } } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java index 0aa07bd9..11818176 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/controller/RedactionController.java @@ -91,6 +91,9 @@ public class RedactionController implements RedactionResource { } else { var table = (Table) t; + if(table == null){ + continue; + } for (var row: table.getRows()){ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java deleted file mode 100644 index dd61063e..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFAreaTextStripper.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.parsing; - -import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; - -import lombok.Getter; -import lombok.Setter; - -import org.apache.pdfbox.text.PDFTextStripperByArea; -import org.apache.pdfbox.text.TextPosition; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class PDFAreaTextStripper extends PDFTextStripperByArea { - - @Getter - private List textPositionSequences = new ArrayList<>(); - - @Setter - private int pageNumber; - - - public PDFAreaTextStripper() throws IOException { - - } - - - @Override - public void writeString(String text, List textPositions) throws IOException { - - int startIndex = 0; - for (int i = 0; i <= textPositions.size() - 1; i++) { - - if (i == 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0"))) { - startIndex++; - continue; - } - - // Strange but sometimes this is happening, for example: Metolachlor2.pdf - if (i > 0 && textPositions.get(i).getX() < textPositions.get(i - 1).getX()) { - List sublist = textPositions.subList(startIndex, i); - if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); - } - startIndex = i; - } - - if (textPositions.get(i).getRotation() == 0 && i > 0 && textPositions.get(i).getX() > textPositions.get(i - 1).getEndX() + 1) { - List sublist = textPositions.subList(startIndex, i); - if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); - } - startIndex = i; - } - - if (i > 0 && (textPositions.get(i).getUnicode().equals(" ") || textPositions.get(i).getUnicode().equals("\u00A0")) && i <= textPositions.size() - 2) { - List sublist = textPositions.subList(startIndex, i); - if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); - } - startIndex = i + 1; - } - } - - List sublist = textPositions.subList(startIndex, textPositions.size()); - if (!sublist.isEmpty() && (sublist.get(sublist.size() - 1).getUnicode().equals(" ") || sublist.get(sublist.size() - 1).getUnicode().equals("\u00A0"))) { - sublist = sublist.subList(0, sublist.size() - 1); - } - if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); - } - super.writeString(text); - } - - - public void clearPositions() { - - textPositionSequences = new ArrayList<>(); - } - -} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java index 3f0f32e1..c2d85d9b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java @@ -195,7 +195,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { @Override - public void writeString(String text, List textPositions) throws IOException { + public void writeString(String text, List textPositions, boolean isParagraphStart) throws IOException { int startIndex = 0; RedTextPosition previous = null; @@ -235,7 +235,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0) .getUnicode() .equals("\t")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); + textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart)); } startIndex = i; } @@ -245,7 +245,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { if (!(sublist.isEmpty() || sublist.size() == 1 && (sublist.get(0).getUnicode().equals(" ") || sublist.get(0).getUnicode().equals("\u00A0") || sublist.get(0) .getUnicode() .equals("\t")))) { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); + textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart)); } startIndex = i; } @@ -265,7 +265,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { textPositionSequences.get(textPositionSequences.size() - 1).add(t); } } else { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); + textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, i == textPositions.size() -1 && isParagraphStart)); } } startIndex = i + 1; @@ -288,7 +288,7 @@ public class PDFLinesTextStripper extends PDFTextStripper { textPositionSequences.get(textPositionSequences.size() - 1).add(t); } } else { - textPositionSequences.add(new TextPositionSequence(sublist, pageNumber)); + textPositionSequences.add(new TextPositionSequence(sublist, pageNumber, isParagraphStart)); } } super.writeString(text); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java index a0eeaaa2..c85b1b32 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFTextStripper.java @@ -27,6 +27,7 @@ import java.text.Bidi; import java.text.Normalizer; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -240,10 +241,10 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { document = doc; output = outputStream; if (getAddMoreFormatting()) { - paragraphEnd = lineSeparator; + paragraphEnd = "\n----ParagraphEnd----\n\n"; pageStart = lineSeparator; - articleStart = lineSeparator; - articleEnd = lineSeparator; + articleStart = "\n----ArticelStart----\n\n"; + articleEnd = "\n----ArticelEnd----\n\n"; } startDocument(document); processPages(document.getPages()); @@ -594,9 +595,14 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { // but this caused a lot of regression test failures. So, I'm leaving it be for // now if (!overlap(positionY, positionHeight, maxYForLine, maxHeightForLine)) { - writeLine(normalize(line)); - line.clear(); + var normalized = normalize(line); +// normalized.stream().filter(l -> System.out.println(l.getText().contains("Plenarprotokoll 20/24")).findFirst().isPresent() + + lastLineStartPosition = handleLineSeparation(current, lastPosition, lastLineStartPosition, maxHeightForLine); + writeLine(normalized, current.isParagraphStart); + line.clear(); + expectedStartOfNextWordX = EXPECTED_START_OF_NEXT_WORD_X_RESET_VALUE; maxYForLine = MAX_Y_FOR_LINE_RESET_VALUE; maxHeightForLine = MAX_HEIGHT_FOR_LINE_RESET_VALUE; @@ -630,7 +636,24 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { if (startOfPage && lastPosition == null) { writeParagraphStart();// not sure this is correct for RTL? } + line.add(new LineItem(position)); + +// Collections.sort(line, new Comparator() { +// +// @Override +// public int compare(LineItem str1, LineItem str2) { +// if(null == str1.getTextPosition()) { +// return 0; +// } +// else if(null == str2.getTextPosition()) { +// return 0; +// } +// return Float.compare(str1.getTextPosition().getX(), str2.getTextPosition().getX()); +// } +// }); + +// line.sort(Comparator.comparing(a -> a.getTextPosition() != null && a.getTextPosition().getX())); } maxHeightForLine = Math.max(maxHeightForLine, positionHeight); minYTopForLine = Math.min(minYTopForLine, positionY - positionHeight); @@ -646,7 +669,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { } // print the final line if (line.size() > 0) { - writeLine(normalize(line)); + writeLine(normalize(line), false); writeParagraphEnd(); } endArticle(); @@ -703,7 +726,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { * @param textPositions The TextPositions belonging to the text. * @throws IOException If there is an error when writing the text. */ - protected void writeString(String text, List textPositions) throws IOException { + protected void writeString(String text, List textPositions, boolean isParagraphEnd) throws IOException { writeString(text); } @@ -1385,6 +1408,7 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { } else { writeLineSeparator(); writeParagraphSeparator(); + lastLineStartPosition.setEndParagraphWritten(); } } else { writeLineSeparator(); @@ -1428,7 +1452,11 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { float newXVal = multiplyFloat(getIndentThreshold(), position.getTextPosition().getWidthOfSpace()); float positionWidth = multiplyFloat(0.25f, position.getTextPosition().getWidth()); - if (yGap > newYVal) { +// if(xGap < 0){ +// result = true; +// } +// else + if (yGap > newYVal) { result = true; } else if (xGap > newXVal) { // text is indented, but try to screen for hanging indent @@ -1636,12 +1664,13 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { * @param line a list with the words of the given line * @throws IOException if something went wrong */ - private void writeLine(List line) throws IOException { + private void writeLine(List line, boolean isParagraphEnd) throws IOException { int numberOfStrings = line.size(); for (int i = 0; i < numberOfStrings; i++) { WordWithTextPositions word = line.get(i); - writeString(word.getText(), word.getTextPositions()); + word.getTextPositions().sort(Comparator.comparing(TextPosition::getX)); + writeString(word.getText(), word.getTextPositions(), isParagraphEnd && i == numberOfStrings - 1); if (i < numberOfStrings - 1) { writeWordSeparator(); } @@ -1963,6 +1992,8 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { private boolean isHangingIndent = false; private boolean isArticleStart = false; + private boolean endParagraphWritten = false; + private TextPosition position = null; @@ -2024,6 +2055,16 @@ public class PDFTextStripper extends LegacyPDFStreamEngine { } + public boolean isEndParagraphWritten() { + + return endParagraphWritten; + } + + public void setEndParagraphWritten(){ + endParagraphWritten = true; + } + + /** * Sets the isArticleStart() flag to true. */ diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java index 0ea72b66..318ac7d8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java @@ -39,6 +39,7 @@ public class TextPositionSequence implements CharSequence { private int rotation; private float pageHeight; private float pageWidth; + private boolean isParagraphStart; public TextPositionSequence(int page) { @@ -47,7 +48,7 @@ public class TextPositionSequence implements CharSequence { } - public TextPositionSequence(List textPositions, int page) { + public TextPositionSequence(List textPositions, int page, boolean isParagraphStart) { this.textPositions = textPositions.stream().map(RedTextPosition::fromTextPosition).collect(Collectors.toList()); this.page = page; @@ -55,6 +56,7 @@ public class TextPositionSequence implements CharSequence { this.rotation = textPositions.get(0).getRotation(); this.pageHeight = textPositions.get(0).getPageHeight(); this.pageWidth = textPositions.get(0).getPageWidth(); + this.isParagraphStart = isParagraphStart; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java index e96a637d..cf30afed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java @@ -10,6 +10,7 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -20,6 +21,7 @@ import org.apache.pdfbox.io.MemoryUsageSetting; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.common.PDRectangle; +import org.apache.pdfbox.util.QuickSort; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.classification.model.Document; @@ -96,7 +98,18 @@ public class PdfSegmentationService { stripper.setStartPage(pageNumber); stripper.setEndPage(pageNumber); stripper.setPdpage(pdPage); - stripper.getText(pdDocument); + stripper.setAddMoreFormatting(true); + stripper.setShouldSeparateByBeads(true); +// stripper.setDropThreshold(0.1f); + var text = stripper.getText(pdDocument); + + System.out.println(text); +// stripper.getTextPositionSequences().forEach(a -> { +// System.out.println(a); +// if(a.isParagraphEnd()){ +// System.out.println("------------------------"); +// } +// }); PDRectangle pdr = pdPage.getMediaBox(); @@ -110,13 +123,35 @@ public class PdfSegmentationService { stripper.getMaxCharHeight()); Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); + +// QuickSort.sort(page.getTextBlocks(), new TextBlockComparator()); +// +// List combined = new ArrayList<>(); +// Iterator itty = page.getTextBlocks().iterator(); +// TextBlock prev = null; +// while (itty.hasNext()){ +// TextBlock current = (TextBlock) itty.next(); +// +// if(prev != null && (((Math.abs(prev.getMaxX() - current.getMaxX()) < 1)|| Math.abs(prev.getMinX() - current.getMinX()) < 1) && current.getMaxY() > prev.getMaxY() && current.getMinY() - prev.getMaxY() < 3)){ +// combined.remove(prev); +// current = prev.union(current); +// } +// +// combined.add(current); +// +// prev= current; +// } +// +// page.setTextBlocks(combined); + + page.setRotation(rotation); page.setLandscape(isLandscape); page.setPageNumber(pageNumber); page.setPageWidth(cropbox.getWidth()); page.setPageHeight(cropbox.getHeight()); - tableExtractionService.extractTables(cleanRulings, page); +// tableExtractionService.extractTables(cleanRulings, page); buildPageStatistics(page); increaseDocumentStatistics(page, document); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TextBlockComparator.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TextBlockComparator.java new file mode 100644 index 00000000..99b820ce --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/TextBlockComparator.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.iqser.red.service.redaction.v1.server.segmentation; + +import java.util.Comparator; + +import org.apache.pdfbox.text.TextPosition; + +import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; +import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; + +/** + * This class is a comparator for TextPosition operators. It handles + * pages with text in different directions by grouping the text based + * on direction and sorting in that direction. This allows continuous text + * in a given direction to be more easily grouped together. + * + * @author Ben Litchfield + */ +public class TextBlockComparator implements Comparator +{ + @Override + public int compare(AbstractTextContainer pos1, AbstractTextContainer pos2) + { + + + // get the text direction adjusted coordinates + float x1 = pos1.getMinX(); + float x2 = pos2.getMinX(); + + float pos1YBottom = pos1.getMaxY(); + float pos2YBottom = pos2.getMaxY(); + + // note that the coordinates have been adjusted so 0,0 is in upper left + float pos1YTop = pos1YBottom - pos1.getHeight(); + float pos2YTop = pos2YBottom - pos2.getHeight(); + + float yDifference = Math.abs(pos1YBottom - pos2YBottom); + + // we will do a simple tolerance comparison + if (yDifference < .1 || + pos2YBottom >= pos1YTop && pos2YBottom <= pos1YBottom || + pos1YBottom >= pos2YTop && pos1YBottom <= pos2YBottom) + { + return Float.compare(x1, x2); + } + else if (pos1YBottom < pos2YBottom) + { + return -1; + } + else + { + return 1; + } + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java index a51ea73a..ad6994f4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfVisualisationService.java @@ -82,14 +82,14 @@ public class PdfVisualisationService { } } - contentStream.setStrokingColor(Color.YELLOW); - contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), - analyzedPage.getBodyTextFrame().getTopLeft().getY(), - analyzedPage.getBodyTextFrame().getWidth(), - analyzedPage.getBodyTextFrame().getHeight()); - - contentStream.stroke(); - +// contentStream.setStrokingColor(Color.YELLOW); +// contentStream.addRect(analyzedPage.getBodyTextFrame().getTopLeft().getX(), +// analyzedPage.getBodyTextFrame().getTopLeft().getY(), +// analyzedPage.getBodyTextFrame().getWidth(), +// analyzedPage.getBodyTextFrame().getHeight()); +// +// contentStream.stroke(); +// contentStream.close(); } } @@ -103,15 +103,15 @@ public class PdfVisualisationService { contentStream.stroke(); if (textBlock.getClassification() != null) { - contentStream.beginText(); - - contentStream.setNonStrokingColor(Color.BLUE); - contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f); - - contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2); - contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir()); - - contentStream.endText(); +// contentStream.beginText(); +// +// contentStream.setNonStrokingColor(Color.BLUE); +// contentStream.setFont(PDType1Font.TIMES_ROMAN, 9f); +// +// contentStream.newLineAtOffset(textBlock.getPdfMinX(), textBlock.getPdfMaxY() + 2); +// contentStream.showText(textBlock.getClassification() + textBlock.getOrientation() + "-->" + textBlock.getSequences().get(0).getDir()); +// +// contentStream.endText(); if (DRAW_POSITIONS) { drawPositions(contentStream, textBlock); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index f968a634..0d75b7b7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -1117,8 +1117,8 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); - String sourcePath ="/tmp"; - String targetPath = "/tmp/result"; + String sourcePath ="/tmp/Input"; + String targetPath = "/tmp/Output"; var files = getFileNames(new HashSet<>(), FileSystems.getDefault().getPath(sourcePath), sourcePath); @@ -1179,7 +1179,7 @@ public class RedactionIntegrationTest { System.out.println("classificationTest"); String tableServiceResponseFile = "files/cv_table_response_VV-511309.json"; - AnalyzeRequest request = prepareStorage("files/new/VV-511309_OCR.pdf", tableServiceResponseFile); + AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId()) @@ -1220,7 +1220,7 @@ public class RedactionIntegrationTest { public void htmlTablesTest() throws IOException { System.out.println("htmlTablesTest"); - AnalyzeRequest request = prepareStorage("files/Minimal Examples/Single Table.pdf"); + AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId()) @@ -1241,7 +1241,7 @@ public class RedactionIntegrationTest { System.out.println("htmlTableRotationTest"); - AnalyzeRequest request = prepareStorage("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf"); + AnalyzeRequest request = prepareStorage("/Users/deiflaender/git/RED/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/Plenarprotokoll 1 (keine Druchsache!) (1).pdf"); RedactionRequest redactionRequest = RedactionRequest.builder() .dossierId(request.getDossierId()) @@ -1254,6 +1254,12 @@ public class RedactionIntegrationTest { try (FileOutputStream fileOutputStream = new FileOutputStream(OsUtils.getTemporaryDirectory() + "/Tables.html")) { fileOutputStream.write(result.getDocument()); } + + + + + + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf similarity index 100% rename from redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table.pdf rename to redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Single Table1.pdf diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/S8Plenar.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/S8Plenar.pdf new file mode 100644 index 00000000..4835b1e5 Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/S8Plenar.pdf differ