Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-264: Avoid phantom cells, by merging line till rounded value of biggest char height/width

Browse Source
This commit is contained in:
deiflaender 2020-09-29 14:21:30 +02:00
parent c9516205fe
commit 11164219c5
6 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/PDFLinesTextStripper.java
View File

@ -44,10 +44,10 @@ import lombok.extern.slf4j.Slf4j;
public class PDFLinesTextStripper extends PDFTextStripper { public class PDFLinesTextStripper extends PDFTextStripper {
@Getter @Getter
private float minCharWidth = Float.MAX_VALUE; private int maxCharWidths;
@Getter @Getter
private float minCharHeight = Float.MAX_VALUE; private int maxCharHeight;
@Getter @Getter
private final List<TextPositionSequence> textPositionSequences = new ArrayList<>(); private final List<TextPositionSequence> textPositionSequences = new ArrayList<>();
@ -201,8 +201,16 @@ public class PDFLinesTextStripper extends PDFTextStripper {
int startIndex = 0; int startIndex = 0;
for (int i = 0; i <= textPositions.size() - 1; i++) { for (int i = 0; i <= textPositions.size() - 1; i++) {
minCharWidth = Math.min(minCharWidth, textPositions.get(i).getWidthDirAdj());
minCharHeight = Math.min(minCharHeight, textPositions.get(i).getHeightDir()); int charHeight = (int) textPositions.get(i).getHeightDir();
if(charHeight > maxCharHeight){
maxCharHeight = charHeight;
}
int charWidth = (int) textPositions.get(i).getWidthDirAdj();
if(charWidth > maxCharWidths){
maxCharWidths = charWidth;
}
if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) { if (i == 0 && textPositions.get(i).getUnicode().equals(" ")) {
startIndex++; startIndex++;
@ -241,8 +249,8 @@ public class PDFLinesTextStripper extends PDFTextStripper {
@Override @Override
public String getText(PDDocument doc) throws IOException { public String getText(PDDocument doc) throws IOException {
minCharWidth = Float.MAX_VALUE; maxCharWidths = 0;
minCharHeight = Float.MAX_VALUE; maxCharWidths = 0;
textPositionSequences.clear(); textPositionSequences.clear();
rulings.clear(); rulings.clear();
graphicsPath.clear(); graphicsPath.clear();

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/ParsedElements.java
View File

@ -17,6 +17,6 @@ public class ParsedElements {
private boolean landscape; private boolean landscape;
private boolean rotated; private boolean rotated;
private float minCharWidth; private float maxCharWidth;
private float minCharHeight; private float maxCharHeight;
} }

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/PdfSegmentationService.java
View File

@ -21,7 +21,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractT
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings; import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService; import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService; import com.iqser.red.service.redaction.v1.server.tableextraction.service.TableExtractionService;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -37,6 +36,7 @@ public class PdfSegmentationService {
private final ClassificationService classificationService; private final ClassificationService classificationService;
private final SectionsBuilderService sectionsBuilderService; private final SectionsBuilderService sectionsBuilderService;
public Document parseDocument(PDDocument pdDocument) throws IOException { public Document parseDocument(PDDocument pdDocument) throws IOException {
Document document = new Document(); Document document = new Document();
@ -56,19 +56,21 @@ public class PdfSegmentationService {
int rotation = pdPage.getRotation(); int rotation = pdPage.getRotation();
boolean isRotated = rotation != 0 && rotation != 360; boolean isRotated = rotation != 0 && rotation != 360;
ParsedElements parsedElements = ParsedElements
.builder() ParsedElements parsedElements = ParsedElements.builder()
.rulings(stripper.getRulings()) .rulings(stripper.getRulings())
.sequences(stripper.getTextPositionSequences()) .sequences(stripper.getTextPositionSequences())
.minCharWidth(Utils.round(stripper.getMinCharWidth(), 2)) .maxCharWidth(stripper.getMaxCharWidths())
.minCharHeight(Utils.round(stripper.getMinCharHeight(), 2)) .maxCharHeight(stripper.getMaxCharWidths())
.landscape(isLandscape) .landscape(isLandscape)
.rotated(isRotated) .rotated(isRotated)
.build(); .build();
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements.getMinCharWidth(), parsedElements.getMinCharHeight()); CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(parsedElements.getRulings(), parsedElements
.getMaxCharWidth(), parsedElements.getMaxCharHeight());
Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings.getVertical()); Page page = blockificationService.blockify(parsedElements.getSequences(), cleanRulings.getHorizontal(), cleanRulings
.getVertical());
page.setRotation(rotation); page.setRotation(rotation);
tableExtractionService.extractTables(cleanRulings, page); tableExtractionService.extractTables(cleanRulings, page);
@ -91,7 +93,10 @@ public class PdfSegmentationService {
} }
private void increaseDocumentStatistics(Page page, Document document) { private void increaseDocumentStatistics(Page page, Document document) {
if (!page.isLandscape()) { if (!page.isLandscape()) {
document.getFontSizeCounter().addAll(page.getFontSizeCounter().getCountPerValue()); document.getFontSizeCounter().addAll(page.getFontSizeCounter().getCountPerValue());
} }
@ -100,6 +105,7 @@ public class PdfSegmentationService {
document.getFontStyleCounter().addAll(page.getFontStyleCounter().getCountPerValue()); document.getFontStyleCounter().addAll(page.getFontStyleCounter().getCountPerValue());
} }
private void buildPageStatistics(Page page) { private void buildPageStatistics(Page page) {
// Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame. // Collect all statistics for the page, except from blocks inside tables, as tables will always be added to BodyTextFrame.

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/service/RulingCleaningService.java
View File

@ -18,9 +18,9 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
@Service @Service
public class RulingCleaningService { public class RulingCleaningService {
public CleanRulings getCleanRulings(List<Ruling> rulings, float minCharWidth, float minCharHeight){ public CleanRulings getCleanRulings(List<Ruling> rulings, float avgCharWidth, float avgCharHeight){
if (!rulings.isEmpty()) { if (!rulings.isEmpty()) {
snapPoints(rulings, minCharWidth , minCharHeight); snapPoints(rulings, avgCharWidth , avgCharHeight);
} }
List<Ruling> vrs = new ArrayList<>(); List<Ruling> vrs = new ArrayList<>();

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -269,7 +269,7 @@ public class RedactionIntegrationTest {
System.out.println("redactionTest"); System.out.println("redactionTest");
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/new/Thiabendazole DAR Addendum for ED_April_2020.pdf"); ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/50 Fludioxonil_RAR_01_Volume_1_2018-02-21.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
@ -388,8 +388,7 @@ public class RedactionIntegrationTest {
public void htmlTablesTest() throws IOException { public void htmlTablesTest() throws IOException {
System.out.println("htmlTablesTest"); System.out.println("htmlTablesTest");
ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 " + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
"Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");
RedactionRequest request = RedactionRequest.builder() RedactionRequest request = RedactionRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream())) .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/Phantom Cells.pdf Normal file
View File

Binary file not shown.