Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

RED-7964: Prototype Visual Layout Parsing:

Browse Source 
added layer for visual layout parsing results (true by default)
added label to drawn boxes
This commit is contained in:
yhampe 2023-12-21 08:54:03 +01:00
parent be2350b289
commit 0f11d901f6
5 changed files with 84 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java
View File

@ -35,6 +35,7 @@ import com.knecon.fforesight.service.layoutparser.processor.python_api.model.ima
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableServiceResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
import com.knecon.fforesight.service.layoutparser.processor.services.BodyTextFrameService;
import com.knecon.fforesight.service.layoutparser.processor.services.RulingCleaningService;
import com.knecon.fforesight.service.layoutparser.processor.services.SectionsBuilderService;
@ -108,7 +109,7 @@ public class LayoutParsingPipeline {
layoutParsingStorageService.storeDocumentData(layoutParsingRequest, DocumentDataMapper.toDocumentData(documentGraph));
layoutParsingStorageService.storeSimplifiedText(layoutParsingRequest, simplifiedSectionTextService.toSimplifiedText(documentGraph));
Map<Integer, List<TableCells>> extractedTableCells = visualLayoutParsingAdapter.buildExtractedTablesPerPage(visualLayoutParsingResponse);
Map<Integer, List<VisualLayoutParsingResult>> extractedTableCells = visualLayoutParsingAdapter.buildExtractedTablesPerPage(visualLayoutParsingResponse);
try (var out = new ByteArrayOutputStream()) {
viewerDocumentService.createViewerDocument(originDocument, documentGraph, out, extractedTableCells, false);
layoutParsingStorageService.storeViewerDocument(layoutParsingRequest, out);

28
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/adapter/VisualLayoutParsingAdapter.java
View File

@ -10,6 +10,7 @@ import org.springframework.stereotype.Service;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingBox;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResponse;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
import lombok.RequiredArgsConstructor;
@ -17,9 +18,9 @@ import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class VisualLayoutParsingAdapter {
public Map<Integer, List<TableCells>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
public Map<Integer, List<VisualLayoutParsingResult>> buildExtractedTablesPerPage(VisualLayoutParsingResponse visualLayoutParsingResponse) {
Map<Integer, List<TableCells>> tableCells = new HashMap<>();
Map<Integer, List<VisualLayoutParsingResult>> tableCells = new HashMap<>();
visualLayoutParsingResponse.getData()
.forEach(tableData -> tableCells.computeIfAbsent(tableData.getPage_idx(), tableCell -> new ArrayList<>()).addAll(convertTableCells(tableData.getBoxes())));
@ -28,21 +29,20 @@ public class VisualLayoutParsingAdapter {
}
public List<TableCells> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
public List<VisualLayoutParsingResult> convertTableCells(List<VisualLayoutParsingBox> tableObjects) {
List<TableCells> parsedTableCells = new ArrayList<>();
List<VisualLayoutParsingResult> parsedTableCells = new ArrayList<>();
tableObjects.stream().forEach(t -> {
if(t.getProbability() > 0.9) {
TableCells tableCells = new TableCells();
tableCells.setX0(t.getBox().getX1());
tableCells.setX1(t.getBox().getX2());
tableCells.setY0(t.getBox().getY1());
tableCells.setY1(t.getBox().getY2());
tableCells.setWidth(tableCells.getX1() - tableCells.getX0());
tableCells.setHeight(tableCells.getY1() - tableCells.getY0());
parsedTableCells.add(tableCells);
}
VisualLayoutParsingResult result = new VisualLayoutParsingResult();
result.setX0(t.getBox().getX1());
result.setX1(t.getBox().getX2());
result.setY0(t.getBox().getY1());
result.setY1(t.getBox().getY2());
result.setWidth(result.getX1() - result.getX0());
result.setHeight(result.getY1() - result.getY0());
result.setLabel(t.getLabel());
parsedTableCells.add(result);
});
return parsedTableCells;

19
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingBoxValue.java Normal file
View File

@ -0,0 +1,19 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class VisualLayoutParsingBoxValue {
private float x1;
private float y1;
private float x2;
private float y2;
}

22
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/python_api/model/table/VisualLayoutParsingResult.java Normal file
View File

@ -0,0 +1,22 @@
package com.knecon.fforesight.service.layoutparser.processor.python_api.model.table;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class VisualLayoutParsingResult {
private float x0;
private float y0;
private float x1;
private float y1;
private float width;
private float height;
private String label;
}

33
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/visualization/ViewerDocumentService.java
View File

@ -34,6 +34,7 @@ import com.knecon.fforesight.service.layoutparser.processor.model.visualization.
import com.knecon.fforesight.service.layoutparser.processor.model.visualization.PlacedText;
import com.knecon.fforesight.service.layoutparser.processor.model.visualization.VisualizationsOnPage;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.TableCells;
import com.knecon.fforesight.service.layoutparser.processor.python_api.model.table.VisualLayoutParsingResult;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
@ -55,7 +56,7 @@ public class ViewerDocumentService {
public void createViewerDocument(PDDocument pdDocument,
Document document,
OutputStream outputStream,
Map<Integer, List<TableCells>> extractedTableCells,
Map<Integer, List<VisualLayoutParsingResult>> extractedTableCells,
boolean layerVisibilityDefaultValue) {
LayoutGrid layoutGrid = layoutGridService.createLayoutGrid(document);
@ -63,6 +64,7 @@ public class ViewerDocumentService {
// If we collect all COSDictionaries we changed and tell it explicitly to only add the changed ones by using saveIncremental it's very fast.
Set<COSDictionary> dictionariesToUpdate = new HashSet<>();
PDOptionalContentGroup layer = addLayerToDocument(pdDocument, dictionariesToUpdate, layerVisibilityDefaultValue);
PDOptionalContentGroup visualLayoutParsingLayer = addLayerToDocument(pdDocument, dictionariesToUpdate, true);
PDFont font = new PDType1Font(Standard14Fonts.FontName.HELVETICA);
for (int pageNumber = 0; pageNumber < pdDocument.getNumberOfPages(); pageNumber++) {
@ -105,11 +107,6 @@ public class ViewerDocumentService {
contentStream.addRect((float) r.getX(), (float) r.getY(), (float) r.getWidth(), (float) r.getHeight());
contentStream.fill();
}
for (TableCells tableCells : extractedTableCells.get(pageNumber)) {
contentStream.setStrokingColor(Color.CYAN);
contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
contentStream.stroke();
}
for (PlacedText placedText : visualizationsOnPage.getPlacedTexts()) {
contentStream.setFont(font, FONT_SIZE);
contentStream.beginText();
@ -126,6 +123,30 @@ public class ViewerDocumentService {
}
contentStream.restoreGraphicsState();
contentStream.endMarkedContent();
contentStream.beginMarkedContent(COSName.OC, visualLayoutParsingLayer);
contentStream.saveGraphicsState();
contentStream.setLineWidth(LINE_WIDTH);
for (VisualLayoutParsingResult tableCells : extractedTableCells.get(pageNumber)) {
contentStream.setStrokingColor(new Color(0xFF0000));
contentStream.addRect((float) tableCells.getX0(), (float) tableCells.getY0(), (float) tableCells.getWidth(), (float) tableCells.getHeight());
contentStream.stroke();
contentStream.setFont(font, FONT_SIZE);
contentStream.beginText();
Matrix textMatrix = new Matrix((float) textDeRotationMatrix.getScaleX(),
(float) textDeRotationMatrix.getShearX(),
(float) textDeRotationMatrix.getShearY(),
(float) textDeRotationMatrix.getScaleY(),
tableCells.getX0() ,
tableCells.getY0());
textMatrix.translate(-((font.getStringWidth(tableCells.getLabel()) / 1000) * FONT_SIZE + (2 * LINE_WIDTH) + 4), -FONT_SIZE);
contentStream.setTextMatrix(textMatrix);
contentStream.showText(tableCells.getLabel());
contentStream.endText();
}
contentStream.restoreGraphicsState();
contentStream.endMarkedContent();
}
dictionariesToUpdate.add(pdPage.getCOSObject());
dictionariesToUpdate.add(pdPage.getResources().getCOSObject());