Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

RED-8670: fix PageContentExtractor

Browse Source
This commit is contained in:
Kilian Schuettler 2025-01-14 14:27:25 +01:00
parent ceb9532d65
commit b307c03f7d
2 changed files with 13 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java
View File

@ -116,24 +116,21 @@ public class LayoutParsingPipeline {
@Value("${LAYOUT_PARSER_VERSION:}") @Value("${LAYOUT_PARSER_VERSION:}")
private String layoutParserVersion; private String layoutParserVersion;
public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException { public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
log.info("Starting layout parsing for {}", layoutParsingRequest.identifier()); log.info("Starting layout parsing for {}", layoutParsingRequest.identifier());
File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId()); File originFile = layoutParsingStorageService.getOriginFile(layoutParsingRequest.originFileStorageId());
File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()) File viewerDocumentFile = layoutParsingStorageService.getViewerDocFile(layoutParsingRequest.viewerDocumentStorageId()).orElse(originFile);
.orElse(originFile);
VisualLayoutParsingResponse visualLayoutParsingResponse = layoutParsingRequest.visualLayoutParsingFileId() VisualLayoutParsingResponse visualLayoutParsingResponse = layoutParsingRequest.visualLayoutParsingFileId()
.map(layoutParsingStorageService::getVisualLayoutParsingFile) .map(layoutParsingStorageService::getVisualLayoutParsingFile).orElse(new VisualLayoutParsingResponse());
.orElse(new VisualLayoutParsingResponse());
ImageServiceResponse imageServiceResponse = layoutParsingRequest.imagesFileStorageId() ImageServiceResponse imageServiceResponse = layoutParsingRequest.imagesFileStorageId()
.map(layoutParsingStorageService::getImagesFile) .map(layoutParsingStorageService::getImagesFile).orElse(new ImageServiceResponse());
.orElse(new ImageServiceResponse());
TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId() TableServiceResponse tableServiceResponse = layoutParsingRequest.tablesFileStorageId()
.map(layoutParsingStorageService::getTablesFile) .map(layoutParsingStorageService::getTablesFile).orElse(new TableServiceResponse());
.orElse(new TableServiceResponse());
IdpResult idpResult = layoutParsingRequest.idpResultStorageId() IdpResult idpResult = layoutParsingRequest.idpResultStorageId()
.map(layoutParsingStorageService::getIdpResultFile).orElse(IdpResult.empty()); .map(layoutParsingStorageService::getIdpResultFile).orElse(IdpResult.empty());
@ -158,10 +155,8 @@ public class LayoutParsingPipeline {
log.info("Storing resulting files for {}", layoutParsingRequest.identifier()); log.info("Storing resulting files for {}", layoutParsingRequest.identifier());
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentWithVisualization.document())); layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentWithVisualization.document()));
if (layoutParsingRequest.documentMarkdownFileStorageId() if (layoutParsingRequest.documentMarkdownFileStorageId().isPresent()) {
.isPresent()) { layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId().get(),
layoutParsingStorageService.storeMarkdownFile(layoutParsingRequest.documentMarkdownFileStorageId()
.get(),
new MarkdownMapper().toMarkdownContent(documentWithVisualization.document())); new MarkdownMapper().toMarkdownContent(documentWithVisualization.document()));
} }
layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentWithVisualization.document())); layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentWithVisualization.document()));
@ -357,8 +352,7 @@ public class LayoutParsingPipeline {
.flatMap(Collection::stream) .flatMap(Collection::stream)
.map(Character::getTextPosition) .map(Character::getTextPosition)
.filter(pos -> pos.getDir().equals(dir)) .filter(pos -> pos.getDir().equals(dir))
.mapToDouble(RedTextPosition::getExactDir).average() .mapToDouble(RedTextPosition::getExactDir).average().orElse(0);
.orElse(0);
if (averageRotation == 0) { if (averageRotation == 0) {
continue; continue;

6
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/PageContentExtractor.java
View File

@ -3,6 +3,7 @@ package com.knecon.fforesight.service.layoutparser.processor.services;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Iterator; import java.util.Iterator;
@ -16,6 +17,7 @@ import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage; import org.apache.pdfbox.pdmodel.PDPage;
import com.google.common.io.Files;
import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox; import com.knecon.fforesight.service.layoutparser.processor.docstrum.model.BoundingBox;
import com.knecon.fforesight.service.layoutparser.processor.model.PageContents; import com.knecon.fforesight.service.layoutparser.processor.model.PageContents;
import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation; import com.knecon.fforesight.service.layoutparser.processor.model.PageInformation;
@ -116,6 +118,8 @@ public class PageContentExtractor {
// As PDFBox caches all types of stuff, we need to close the document every once in a while to save on RAM // As PDFBox caches all types of stuff, we need to close the document every once in a while to save on RAM
doc.close(); doc.close();
doc = openDocument(document); doc = openDocument(document);
pageGetter = new PageGetter(doc.getPages()
.iterator(), pageCount);
} }
extractPage(pageNumber, doc, pageGetter.getPage(pageNumber)); extractPage(pageNumber, doc, pageGetter.getPage(pageNumber));
@ -136,6 +140,7 @@ public class PageContentExtractor {
@SneakyThrows @SneakyThrows
public void extractPage(Integer pageNumber, PDDocument doc, PDPage pdPage) { public void extractPage(Integer pageNumber, PDDocument doc, PDPage pdPage) {
PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
PDFLinesTextStripper stripper = new PDFLinesTextStripper(); PDFLinesTextStripper stripper = new PDFLinesTextStripper();
stripper.setPageNumber(pageNumber); stripper.setPageNumber(pageNumber);
stripper.setStartPage(pageNumber); stripper.setStartPage(pageNumber);
@ -143,7 +148,6 @@ public class PageContentExtractor {
stripper.setPdpage(pdPage); stripper.setPdpage(pdPage);
stripper.getText(doc); stripper.getText(doc);
PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
List<Word> words = stripper.getWords(); List<Word> words = stripper.getWords();
List<Ruling> rulings = stripper.getRulings(); List<Ruling> rulings = stripper.getRulings();
List<Box> graphicBBoxes = findGraphicBBoxes(pageInformation, pdPage, doc, words); List<Box> graphicBBoxes = findGraphicBBoxes(pageInformation, pdPage, doc, words);