diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java index 1a7b2e0e..3cfcee36 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java @@ -22,7 +22,7 @@ public abstract class AbstractPageBlock { @JsonIgnore protected float maxY; @JsonIgnore - protected String classification; + protected PageBlockType classification; @JsonIgnore protected int page; @@ -35,7 +35,7 @@ public abstract class AbstractPageBlock { public boolean isHeadline() { - return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().startsWith("H"); + return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().isHeadline(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java new file mode 100644 index 00000000..16c52c0f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/PageBlockType.java @@ -0,0 +1,34 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model; + +public enum PageBlockType { + H1, + H2, + H3, + H4, + HEADER, + FOOTER, + TITLE, + PARAGRAPH, + PARAGRAPH_BOLD, + PARAGRAPH_ITALIC, + PARAGRAPH_UNKNOWN, + OTHER, + TABLE; + + + public static PageBlockType getHeadlineType(int i) { + + return switch (i) { + case 1 -> PageBlockType.H1; + case 2 -> PageBlockType.H2; + case 3 -> PageBlockType.H3; + default -> PageBlockType.H4; + }; + } + + + public boolean isHeadline() { + + return this.equals(H1) || this.equals(H2) || this.equals(H3) || this.equals(H4); + } +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java index bb39eb92..86df0526 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/table/TablePageBlock.java @@ -12,6 +12,7 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.CollectionUtils; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; import lombok.Getter; @@ -39,7 +40,7 @@ public class TablePageBlock extends AbstractPageBlock { minY = area.getBottom(); maxX = area.getRight(); maxY = area.getTop(); - classification = "TablePageBlock"; + classification = PageBlockType.TABLE; this.rotation = rotation; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java index 531a9429..dbb07713 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/ClassificationTextBlock.java @@ -5,6 +5,7 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnore; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; import lombok.AllArgsConstructor; @@ -45,7 +46,7 @@ public class ClassificationTextBlock extends AbstractPageBlock { private float highestFontSize; @JsonIgnore - private String classification; + private PageBlockType classification; @JsonIgnore diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java index f4fe3736..94b83c80 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BlockificationService.java @@ -40,7 +40,7 @@ public class BlockificationService { int indexOnPage = 0; List chunkWords = new ArrayList<>(); - List chunkBlockList1 = new ArrayList<>(); + List chunkBlockList = new ArrayList<>(); float minX = 1000, maxX = 0, minY = 1000, maxY = 0; TextPositionSequence prev = null; @@ -60,14 +60,14 @@ public class BlockificationService { if (prev != null && (lineSeparation || startFromTop || splitByX || splitByDir || isSplitByRuling)) { Orientation prevOrientation = null; - if (!chunkBlockList1.isEmpty()) { - prevOrientation = chunkBlockList1.get(chunkBlockList1.size() - 1).getOrientation(); + if (!chunkBlockList.isEmpty()) { + prevOrientation = chunkBlockList.get(chunkBlockList.size() - 1).getOrientation(); } ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); indexOnPage++; - chunkBlockList1.add(cb1); + chunkBlockList.add(cb1); chunkWords = new ArrayList<>(); if (splitByX && !isSplitByRuling) { @@ -108,10 +108,10 @@ public class BlockificationService { ClassificationTextBlock cb1 = buildTextBlock(chunkWords, indexOnPage); if (cb1 != null) { - chunkBlockList1.add(cb1); + chunkBlockList.add(cb1); } - Iterator itty = chunkBlockList1.iterator(); + Iterator itty = chunkBlockList.iterator(); ClassificationTextBlock previousLeft = null; ClassificationTextBlock previousRight = null; @@ -141,7 +141,7 @@ public class BlockificationService { } } - itty = chunkBlockList1.iterator(); + itty = chunkBlockList.iterator(); ClassificationTextBlock previous = null; while (itty.hasNext()) { ClassificationTextBlock block = (ClassificationTextBlock) itty.next(); @@ -157,7 +157,7 @@ public class BlockificationService { previous = block; } - return new ClassificationPage(chunkBlockList1); + return new ClassificationPage(chunkBlockList); } @@ -254,7 +254,7 @@ public class BlockificationService { verticalRulingLines, word.getDir().getDegrees(), word.getPageWidth(), - word.getPageHeight()); // + word.getPageHeight()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java index bb7e8821..562ecd94 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/BodyTextFrameService.java @@ -17,6 +17,9 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.ut @Service public class BodyTextFrameService { + private static final float APPROXIMATE_HEADER_LINE_COUNT = 2.9f; + + /** * Adjusts and sets the body text frame to a page. * Note: This needs to use Pdf Coordinate System where {0,0} rotated with the page rotation. @@ -84,7 +87,7 @@ public class BodyTextFrameService { } float approxLineCount = PositionUtils.getApproxLineCount(textBlock); - if (approxLineCount < 2.9f) { + if (approxLineCount < APPROXIMATE_HEADER_LINE_COUNT) { continue; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java index f80d786b..af75d42f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/ClassificationService.java @@ -9,6 +9,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationDocument; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.utils.PositionUtils; @@ -53,21 +54,21 @@ public class ClassificationService { var bodyTextFrame = page.getBodyTextFrame(); if (document.getFontSizeCounter().getMostPopular() == null) { - textBlock.setClassification("Other"); + textBlock.setClassification(PageBlockType.OTHER); return; } if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { - textBlock.setClassification("Header"); + textBlock.setClassification(PageBlockType.HEADER); } else if (PositionUtils.isUnderBodyTextFrame(bodyTextFrame, textBlock, page.getRotation()) && (document.getFontSizeCounter() .getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter().getMostPopular())) { - textBlock.setClassification("Footer"); + textBlock.setClassification(PageBlockType.FOOTER); } else if (page.getPageNumber() == 1 && (PositionUtils.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter().getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter().getMostPopular() || page.getTextBlocks() .size() == 1)) { if (!Pattern.matches("[0-9]+", textBlock.toString())) { - textBlock.setClassification("Title"); + textBlock.setClassification(PageBlockType.TITLE); } } else if (textBlock.getMostPopularWordFontSize() > document.getFontSizeCounter() .getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle().equals("bold") || !document.getFontStyleCounter() @@ -80,36 +81,34 @@ public class ClassificationService { for (int i = 1; i <= headlineFontSizes.size(); i++) { if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) { - textBlock.setClassification("H " + i); + textBlock.setClassification(PageBlockType.getHeadlineType(i)); document.setHeadlines(true); } } - } else if (!textBlock.getText().startsWith("TablePageBlock ") && !textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, - textBlock) && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter() - .getMostPopular() - .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() + } else if (!textBlock.getText().startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle() + .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences() .get(0) .getTextPositions() .get(0) .getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) { - textBlock.setClassification("H " + (headlineFontSizes.size() + 1)); + textBlock.setClassification(PageBlockType.getHeadlineType(headlineFontSizes.size() + 1)); document.setHeadlines(true); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() .getMostPopular() && textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) { - textBlock.setClassification("TextBlock Bold"); + textBlock.setClassification(PageBlockType.PARAGRAPH_BOLD); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont() .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle() .equals(document.getFontStyleCounter().getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter().getMostPopular()) { - textBlock.setClassification("TextBlock"); + textBlock.setClassification(PageBlockType.PARAGRAPH); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter() .getMostPopular() && textBlock.getMostPopularWordStyle().equals("italic") && !document.getFontStyleCounter() .getMostPopular() .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) { - textBlock.setClassification("TextBlock Italic"); + textBlock.setClassification(PageBlockType.PARAGRAPH_ITALIC); } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) { - textBlock.setClassification("TextBlock Unknown"); + textBlock.setClassification(PageBlockType.PARAGRAPH_UNKNOWN); } else { - textBlock.setClassification("Other"); + textBlock.setClassification(PageBlockType.OTHER); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java index 1b33d464..a6c79da6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/service/SectionsBuilderService.java @@ -16,6 +16,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationHeader; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationSection; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.PageBlockType; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.image.ClassifiedImage; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.Cell; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; @@ -52,22 +53,22 @@ public class SectionsBuilderService { current.setPage(page.getPageNumber()); - if (current.getClassification().equals("Header")) { + if (current.getClassification().equals(PageBlockType.HEADER)) { header.add((ClassificationTextBlock) current); continue; } - if (current.getClassification().equals("Footer")) { + if (current.getClassification().equals(PageBlockType.FOOTER)) { footer.add((ClassificationTextBlock) current); continue; } - if (current.getClassification().equals("Other")) { + if (current.getClassification().equals(PageBlockType.OTHER)) { unclassifiedText.add((ClassificationTextBlock) current); continue; } - if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) { + if (prev != null && current.getClassification().isHeadline() && !prev.getClassification().isHeadline() || !document.isHeadlines()) { ClassificationSection chunkBlock = buildTextBlock(chunkWords, lastHeadline); chunkBlock.setHeadline(lastHeadline); if (document.isHeadlines()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java index 52eec75e..37790108 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java @@ -93,7 +93,7 @@ public class DocumentGraphMapper { } else { pages.forEach(page -> page.getMainBody().add(node)); } - newEntries.add(DocumentTree.Entry.builder().treeId(treeId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build()); + newEntries.add(DocumentTree.Entry.builder().treeId(treeId).children(buildEntries(entryData.getSubEntries(), context)).node(node).build()); } return newEntries; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java index ff656377..b6a5fc63 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java @@ -26,7 +26,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Header; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Headline; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; @@ -87,7 +86,7 @@ public class DocumentGraphFactory { List textBlocks = new LinkedList<>(textBlocksToMerge); textBlocks.add(originalTextBlock); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); - List treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node.getType(), node); + List treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node); node.setLeafTextBlock(textBlock); node.setTreeId(treeId); } @@ -107,7 +106,7 @@ public class DocumentGraphFactory { .build(); page.getMainBody().add(imageNode); - List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, NodeType.IMAGE, imageNode); + List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(section, imageNode); imageNode.setTreeId(tocId); } @@ -152,7 +151,7 @@ public class DocumentGraphFactory { footer, context, page); - List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.FOOTER, footer); + List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer); footer.setTreeId(tocId); footer.setLeafTextBlock(textBlock); page.setFooter(footer); @@ -164,7 +163,7 @@ public class DocumentGraphFactory { Page page = context.getPage(textBlocks.get(0).getPage()); Header header = Header.builder().documentTree(context.getDocumentTree()).build(); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page); - List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.HEADER, header); + List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header); header.setTreeId(tocId); header.setLeafTextBlock(textBlock); page.setHeader(header); @@ -176,7 +175,7 @@ public class DocumentGraphFactory { Page page = context.getPage(pageIndex); Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page); - List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.FOOTER, footer); + List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer); footer.setTreeId(tocId); footer.setLeafTextBlock(textBlock); page.setFooter(footer); @@ -188,7 +187,7 @@ public class DocumentGraphFactory { Page page = context.getPage(pageIndex); Header header = Header.builder().documentTree(context.getDocumentTree()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page); - List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.HEADER, header); + List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header); header.setTreeId(tocId); header.setLeafTextBlock(textBlock); page.setHeader(header); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java index 3b60e4ef..902b9d36 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java @@ -44,19 +44,19 @@ public class SearchTextWithTextPositionFactory { } previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build(); - context.sb.append(" "); + context.stringBuilder.append(" "); context.stringIdxToPositionIdx.add(context.positionIdx); ++context.stringIdx; } - assert context.sb.length() == context.stringIdxToPositionIdx.size(); + assert context.stringBuilder.length() == context.stringIdxToPositionIdx.size(); List positions = sequences.stream() .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence))) .toList(); return SearchTextWithTextPositionDto.builder() - .searchText(context.sb.toString()) + .searchText(context.stringBuilder.toString()) .lineBreaks(context.lineBreaksStringIdx) .stringCoordsToPositionCoords(context.stringIdxToPositionIdx) .positions(positions) @@ -66,7 +66,7 @@ public class SearchTextWithTextPositionFactory { private static void appendCurrentTextPosition(Context context, RedTextPosition currentTextPosition) { - context.sb.append(currentTextPosition.getUnicode()); + context.stringBuilder.append(currentTextPosition.getUnicode()); // unicode characters with more than 16-bit encoding have a length > 1 in java strings for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) { @@ -81,7 +81,7 @@ public class SearchTextWithTextPositionFactory { if (isLineBreak(currentTextPosition, previousTextPosition)) { if (lastHyphenDirectlyBeforeLineBreak(context)) { - context.sb.delete(context.lastHyphenIdx, context.sb.length()); + context.stringBuilder.delete(context.lastHyphenIdx, context.stringBuilder.length()); context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx); context.stringIdx = context.lastHyphenIdx; context.lastHyphenIdx = -3; @@ -166,14 +166,14 @@ public class SearchTextWithTextPositionFactory { List stringIdxToPositionIdx = new LinkedList<>(); List lineBreaksStringIdx = new LinkedList<>(); - StringBuilder sb = new StringBuilder(); + StringBuilder stringBuilder = new StringBuilder(); int stringIdx; int positionIdx; // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away. // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height. // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate - // This is why, we need to initialize this to <= -3, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there. + // This is why, we need to initialize this to < -2, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there. // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3. int lastHyphenIdx = -3; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java index a6053f92..37374f5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java @@ -14,7 +14,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.ClassificationTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.TableMergingUtility; @@ -54,9 +53,9 @@ public class SectionNodeFactory { private static List getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, Section section) { if (parentNode == null) { - return context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.SECTION, section); + return context.getDocumentTree().createNewMainEntryAndReturnId(section); } else { - return context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.SECTION, section); + return context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, section); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java index b89f1bc7..1d8b612a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java @@ -12,7 +12,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.table.TablePageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.GenericSemanticNode; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; @@ -37,7 +36,7 @@ public class TableNodeFactory { pages.forEach(page -> addTableToPage(page, parentNode, table)); - List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.TABLE, table); + List tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, table); table.setTreeId(tocId); addTableCells(mergedRows, table, context); @@ -104,7 +103,7 @@ public class TableNodeFactory { TextBlock textBlock; - List tocId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, NodeType.TABLE_CELL, tableCell); + List tocId = context.getDocumentTree().createNewTableChildEntryAndReturnId(tableNode, tableCell); tableCell.setTreeId(tocId); if (cell.getTextBlocks().isEmpty()) { @@ -142,8 +141,7 @@ public class TableNodeFactory { private boolean firstTextBlockIsHeadline(Cell cell) { - String classification = cell.getTextBlocks().get(0).getClassification(); - return classification != null && classification.startsWith("H"); + return cell.getTextBlocks().get(0).isHeadline(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java index 77c358a8..6ca32f5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/DocumentTree.java @@ -33,7 +33,7 @@ public class DocumentTree { public DocumentTree(Document document) { - root = Entry.builder().treeId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(document).build(); + root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build(); } @@ -43,32 +43,32 @@ public class DocumentTree { } - public List createNewMainEntryAndReturnId(NodeType nodeType, GenericSemanticNode node) { + public List createNewMainEntryAndReturnId(GenericSemanticNode node) { - return createNewChildEntryAndReturnIdImpl(Collections.emptyList(), nodeType, node); + return createNewChildEntryAndReturnIdImpl(Collections.emptyList(), node); } - public List createNewChildEntryAndReturnId(GenericSemanticNode parentNode, NodeType nodeType, GenericSemanticNode node) { + public List createNewChildEntryAndReturnId(GenericSemanticNode parentNode, GenericSemanticNode node) { - return createNewChildEntryAndReturnIdImpl(parentNode.getTreeId(), nodeType, node); + return createNewChildEntryAndReturnIdImpl(parentNode.getTreeId(), node); } - public List createNewChildEntryAndReturnId(GenericSemanticNode parentNode, NodeType nodeType, Table node) { + public List createNewChildEntryAndReturnId(GenericSemanticNode parentNode, Table node) { - return createNewChildEntryAndReturnIdImpl(parentNode.getTreeId(), nodeType, node); + return createNewChildEntryAndReturnIdImpl(parentNode.getTreeId(), node); } - public List createNewTableChildEntryAndReturnId(Table parentTable, NodeType nodeType, TableCell tableCell) { + public List createNewTableChildEntryAndReturnId(Table parentTable, TableCell tableCell) { - return createNewChildEntryAndReturnIdImpl(parentTable.getTreeId(), nodeType, tableCell); + return createNewChildEntryAndReturnIdImpl(parentTable.getTreeId(), tableCell); } @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong - private List createNewChildEntryAndReturnIdImpl(List parentId, NodeType nodeType, SemanticNode node) { + private List createNewChildEntryAndReturnIdImpl(List parentId, SemanticNode node) { if (!entryExists(parentId)) { throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId)); @@ -77,7 +77,7 @@ public class DocumentTree { Entry parent = getEntryById(parentId); List newId = new LinkedList<>(parentId); newId.add(parent.children.size()); - parent.children.add(Entry.builder().treeId(newId).node(node).type(nodeType).children(new LinkedList<>()).build()); + parent.children.add(Entry.builder().treeId(newId).node(node).children(new LinkedList<>()).build()); return newId; } @@ -119,7 +119,7 @@ public class DocumentTree { public Stream streamChildNodesOfType(List treeId, NodeType nodeType) { - return getEntryById(treeId).children.stream().filter(entry -> entry.type.equals(nodeType)).map(Entry::getNode); + return getEntryById(treeId).children.stream().filter(entry -> entry.node.getType().equals(nodeType)).map(Entry::getNode); } @@ -195,7 +195,6 @@ public class DocumentTree { public static class Entry { List treeId; - NodeType type; SemanticNode node; List children; @@ -207,6 +206,12 @@ public class DocumentTree { } + public NodeType getType() { + + return node.getType(); + } + + @Override public int hashCode() {