Marmelator/layout-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 289 Wiki Activity

RED-8825: minor fixes

Browse Source 
* also added overrides via env variables
This commit is contained in:
Kilian Schuettler 2024-05-07 17:37:42 +02:00
parent e935cc7b14
commit 6a65d7f9fc
5 changed files with 38 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutParsingPipeline.java
View File

@ -96,6 +96,7 @@ public class LayoutParsingPipeline {
VisualLayoutParsingAdapter visualLayoutParsingAdapter;
ClarifyndClassificationService clarifyndClassificationService;
GraphicExtractorService graphicExtractorService;
LayoutparserSettings settings;
public LayoutParsingFinishedEvent parseLayoutAndSaveFilesToStorage(LayoutParsingRequest layoutParsingRequest) throws IOException {
@ -121,7 +122,8 @@ public class LayoutParsingPipeline {
tableServiceResponse = layoutParsingStorageService.getTablesFile(layoutParsingRequest.tablesFileStorageId().get());
}
ClassificationDocument classificationDocument = parseLayout(layoutParsingRequest.layoutParsingType(),
ClassificationDocument classificationDocument = parseLayout(settings.getLayoutParsingTypeOverride() == null //
? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(),
originFile,
imageServiceResponse,
tableServiceResponse,
@ -130,7 +132,8 @@ public class LayoutParsingPipeline {
log.info("Building document graph for {}", layoutParsingRequest.identifier());
Document documentGraph = observeBuildDocumentGraph(layoutParsingRequest.layoutParsingType(), classificationDocument);
Document documentGraph = observeBuildDocumentGraph(settings.getLayoutParsingTypeOverride() == null //
? layoutParsingRequest.layoutParsingType() : settings.getLayoutParsingTypeOverride(), classificationDocument);
log.info("Creating viewer document for {}", layoutParsingRequest.identifier());
@ -224,7 +227,9 @@ public class LayoutParsingPipeline {
Map<Integer, List<ClassifiedImage>> signatures = visualLayoutParsingAdapter.buildExtractedSignaturesPerPage(visualLayoutParsingResponse);
ClassificationDocument classificationDocument = new ClassificationDocument();
classificationDocument.getVisualizations().setActive(identifier.containsKey("debug"));
if (settings.isDebug() || identifier.containsKey("debug")) {
classificationDocument.getVisualizations().setActive(true);
}
List<ClassificationPage> classificationPages = new ArrayList<>();
@ -266,7 +271,8 @@ public class LayoutParsingPipeline {
classificationDocument.getVisualizations().addRulingVisualization(stripper.getRulings(), pageNumber);
CleanRulings cleanRulings = rulingCleaningService.deduplicateAndStraightenRulings(pdfTableCells.get(pageNumber), stripper.getRulings());
List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), PageInformation.fromPDPage(pageNumber, pdPage));
PageInformation pageInformation = PageInformation.fromPDPage(pageNumber, pdPage);
List<Cell> emptyTableCells = TableExtractionService.findCells(cleanRulings.getHorizontals(), cleanRulings.getVerticals(), pageInformation);
classificationDocument.getVisualizations().addCellVisualizations(emptyTableCells, pageNumber);
TextRulingsClassifier.classifyUnderlinedAndStrikethroughText(words, cleanRulings);

20
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/LayoutparserSettings.java Normal file
View File

@ -0,0 +1,20 @@
package com.knecon.fforesight.service.layoutparser.processor;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import com.knecon.fforesight.service.layoutparser.internal.api.queue.LayoutParsingType;
import lombok.AccessLevel;
import lombok.Data;
import lombok.experimental.FieldDefaults;
@Data
@Configuration
@ConfigurationProperties("layoutparser")
@FieldDefaults(level = AccessLevel.PRIVATE)
public class LayoutparserSettings {
boolean debug;
LayoutParsingType layoutParsingTypeOverride;
}

7
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/docstrum/model/ColumnDetector.java
View File

@ -15,11 +15,12 @@ WIP, mostly working, needs to be tested a bit more
public class ColumnDetector {
public static final double MAX_VALUE_THRESHOLD = 0.5;
final static int bins_num = 128;
final static int bins_num = 512;
final static int globalStartIdx = 0; // ignore outer parts completely, we don't expect columns there
final static int globalEndIdx = bins_num; // i chose 7, since thirds seems a likely split for columns, therefore divided by 6 would eliminate those.
public static final double DERIVATIVE_ZERO_THRESHOLD = 1e-10;
public static final double MINIMUM_THRESHOLD_FOR_COLUMNS = 0.05;
public static final double NEAR_GLOBAL_THRESHOLD = 0.5;
double minY;
double maxY;
double midY;
@ -241,10 +242,10 @@ public class ColumnDetector {
List<Integer> nearGlobalDvMaximaIdx = new LinkedList<>();
List<Integer> nearGlobalDvMinimaIdx = new LinkedList<>();
for (int i = globalStartIdx; i < globalEndIdx; i++) {
if (derivative[i] <= minDvValue * 0.8) {
if (derivative[i] <= minDvValue * NEAR_GLOBAL_THRESHOLD) {
nearGlobalDvMinimaIdx.add(i);
}
if (derivative[i] >= maxDvValue * 0.8) {
if (derivative[i] >= maxDvValue * NEAR_GLOBAL_THRESHOLD) {
nearGlobalDvMaximaIdx.add(i);
}
}

4
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/model/text/RedTextPosition.java
View File

@ -76,9 +76,9 @@ public class RedTextPosition extends BoundingBox {
pos.setBBoxDirAdj(dirAdjPosition);
AffineTransform affineTransform = getRotationMatrix(TextDirection.fromDegrees(textPosition.getDir()), textPosition.getPageWidth(), textPosition.getPageHeight());
Rectangle2D initialUserSpacePositionRect = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
Rectangle2D bBoxInitialUserSpace = affineTransform.createTransformedShape(dirAdjPosition).getBounds2D();
pos.setBBoxInitialUserSpace(initialUserSpacePositionRect); // These are definitely correct
pos.setBBoxInitialUserSpace(bBoxInitialUserSpace); // These are definitely correct
return pos;
}

2
layoutparser-service/layoutparser-service-processor/src/main/java/com/knecon/fforesight/service/layoutparser/processor/services/RulingCleaningService.java
View File

@ -58,6 +58,7 @@ public class RulingCleaningService {
.toList());
List<Ruling> cleanedVerticalRulings = groupedOverlappingVerticalRectangles.stream()
.map(rectList -> getXCenteredRuling(RectangleTransformations.rectangle2DBBox(rectList)))
.filter(ruling -> ruling.length() > 0)
.toList();
List<List<Rectangle2D>> groupedOverlappingHorizontalRectangles = groupOverlappingRectangles(rulings.horizontalLines.stream()
@ -67,6 +68,7 @@ public class RulingCleaningService {
List<Ruling> cleanedHorizontalRulings = groupedOverlappingHorizontalRectangles.stream()
.map(rectList -> getYCenteredRuling(RectangleTransformations.rectangle2DBBox(rectList)))
.filter(ruling -> ruling.length() > 0)
.collect(Collectors.toList());
return new Rulings(cleanedVerticalRulings, cleanedHorizontalRulings);