diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java index 7ade3b26..1a7b2e0e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/AbstractPageBlock.java @@ -33,6 +33,12 @@ public abstract class AbstractPageBlock { public abstract String getText(); + public boolean isHeadline() { + + return this instanceof ClassificationTextBlock && this.getClassification() != null && this.getClassification().startsWith("H"); + } + + public boolean containsBlock(ClassificationTextBlock other) { return this.minX <= other.getPdfMinX() && this.maxX >= other.getPdfMaxX() && this.minY >= other.getPdfMinY() && this.maxY <= other.getPdfMaxY(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SearchableText.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SearchableText.java index f5144d92..36677e1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SearchableText.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/model/text/SearchableText.java @@ -36,10 +36,14 @@ public class SearchableText { StringBuilder sb = new StringBuilder(); for (TextPositionSequence word : sequences) { - sb.append(word.toString()); + sb.append(word); sb.append(' '); } - return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll(" {2}", " "); + String text = sb.toString(); + text = TextNormalizationUtilities.removeHyphenLineBreaks(text); + text = TextNormalizationUtilities.removeLineBreaks(text); + text = TextNormalizationUtilities.removeRepeatingWhitespaces(text); + return text; } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java index 71a1bdd4..e346266f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicPositionBlockData.java @@ -27,21 +27,11 @@ public class AtomicPositionBlockData { return AtomicPositionBlockData.builder() .id(atomicTextBlock.getId()) .positions(toPrimitiveFloatMatrix(atomicTextBlock.getPositions())) - .stringIdxToPositionIdx(toPrimitiveIntArray(atomicTextBlock.getStringIdxToPositionIdx())) + .stringIdxToPositionIdx(atomicTextBlock.getStringIdxToPositionIdx().stream().mapToInt(Integer::intValue).toArray()) .build(); } - private static int[] toPrimitiveIntArray(List list) { - - int[] array = new int[list.size()]; - for (int i = 0; i < list.size(); i++) { - array[i] = list.get(i); - } - return array; - } - - private static float[][] toPrimitiveFloatMatrix(List positions) { float[][] positionMatrix = new float[positions.size()][]; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java index 804a7f5f..7afeabd4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/AtomicTextBlockData.java @@ -1,7 +1,5 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data; -import java.util.List; - import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import lombok.AccessLevel; @@ -34,18 +32,8 @@ public class AtomicTextBlockData { .numberOnPage(atomicTextBlock.getNumberOnPage()) .start(atomicTextBlock.getBoundary().start()) .end(atomicTextBlock.getBoundary().end()) - .lineBreaks(toPrimitiveIntArray(atomicTextBlock.getLineBreaks())) + .lineBreaks(atomicTextBlock.getLineBreaks().stream().mapToInt(Integer::intValue).toArray()) .build(); } - - private static int[] toPrimitiveIntArray(List list) { - - int[] array = new int[list.size()]; - for (int i = 0; i < list.size(); i++) { - array[i] = list.get(i); - } - return array; - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java index b960e935..3997d6b5 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentData.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; + import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -17,4 +19,25 @@ public class DocumentData { AtomicPositionBlockData[] atomicPositionBlocks; DocumentTreeData documentTreeData; + + public static DocumentData fromDocument(Document document) { + + var atomicPositionBlocks = document.streamTerminalTextBlocksInOrder() + .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) + .distinct() + .map(AtomicPositionBlockData::fromAtomicTextBlock) + .toArray(AtomicPositionBlockData[]::new); + + var atomicTextBlocks = document.streamTerminalTextBlocksInOrder() + .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) + .distinct() + .map(AtomicTextBlockData::fromAtomicTextBlock) + .toArray(AtomicTextBlockData[]::new); + + var pages = document.getPages().stream().map(PageData::fromPage).toArray(PageData[]::new); + + var documentTreeData = new DocumentTreeData(DocumentTreeData.EntryData.fromEntry(document.getDocumentTree().getRoot())); + return new DocumentData(pages, atomicTextBlocks, atomicPositionBlocks, documentTreeData); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java index 61ccb956..bb156830 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/DocumentTreeData.java @@ -1,10 +1,19 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -63,12 +72,40 @@ public class DocumentTreeData { NodeType type; int[] treeId; - Long[] atomicBlocks; - Long[] pages; + Long[] atomicBlockIds; + Long[] pageNumbers; Map properties; List subEntries; + public static EntryData fromEntry(DocumentTree.Entry entry) { + + Long[] atomicBlockIds = toAtomicTextBlockIds(entry); + + Map properties = switch (entry.getType()) { + case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode()); + case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode()); + case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode()); + case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode()); + default -> new HashMap<>(); + }; + var treeId = entry.getTreeId().stream().mapToInt(Integer::intValue).toArray(); + var pageNumbers = entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new); + var subEntries = entry.getChildren().stream().map(EntryData::fromEntry).toList(); + return new EntryData(entry.getType(), treeId, atomicBlockIds, pageNumbers, properties, subEntries); + } + + + private static Long[] toAtomicTextBlockIds(DocumentTree.Entry entry) { + + if (entry.getNode().isLeaf()) { + return entry.getNode().getLeafTextBlock().getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new); + } else { + return new Long[]{}; + } + } + + @Override public String toString() { @@ -83,7 +120,7 @@ public class DocumentTreeData { sb.append(type); sb.append(" atbs = "); - sb.append(atomicBlocks.length); + sb.append(atomicBlockIds.length); return sb.toString(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java index f9db6987..75bd3166 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/PageData.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; + import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; @@ -17,4 +19,10 @@ public class PageData { int width; int rotation; + + public static PageData fromPage(Page page) { + + return new PageData(page.getNumber(), page.getHeight(), page.getWidth(), page.getRotation()); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java deleted file mode 100644 index 4ba1e33c..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentDataMapper.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentTreeData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; - -import lombok.experimental.UtilityClass; - -@UtilityClass -public class DocumentDataMapper { - - public DocumentData toDocumentData(Document document) { - - List atomicTextBlockData = document.streamTerminalTextBlocksInOrder() - .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) - .distinct() - .map(AtomicTextBlockData::fromAtomicTextBlock) - .toList(); - - List atomicPositionBlockData = document.streamTerminalTextBlocksInOrder() - .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) - .distinct() - .map(AtomicPositionBlockData::fromAtomicTextBlock) - .toList(); - - List pageData = document.getPages().stream().map(DocumentDataMapper::toPageData).toList(); - DocumentTreeData documentTreeData = toTableOfContentsData(document.getDocumentTree()); - return DocumentData.builder() - .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0])) - .atomicPositionBlocks(atomicPositionBlockData.toArray(new AtomicPositionBlockData[0])) - .pages(pageData.toArray(new PageData[0])) - .documentTreeData(documentTreeData) - .build(); - } - - - private DocumentTreeData toTableOfContentsData(DocumentTree documentTree) { - - return new DocumentTreeData(toEntryData(documentTree.getRoot())); - } - - - private DocumentTreeData.EntryData toEntryData(DocumentTree.Entry entry) { - - Long[] atomicTextBlocks; - - if (entry.getNode().isLeaf()) { - atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getLeafTextBlock()); - } else { - atomicTextBlocks = new Long[]{}; - } - - Map properties = switch (entry.getType()) { - case TABLE -> PropertiesMapper.buildTableProperties((Table) entry.getNode()); - case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCell) entry.getNode()); - case IMAGE -> PropertiesMapper.buildImageProperties((Image) entry.getNode()); - case SECTION -> PropertiesMapper.buildSectionProperties((Section) entry.getNode()); - default -> new HashMap<>(); - }; - - return DocumentTreeData.EntryData.builder() - .treeId(toPrimitiveIntArray(entry.getTreeId())) - .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList()) - .type(entry.getType()) - .atomicBlocks(atomicTextBlocks) - .pages(entry.getNode().getPages().stream().map(Page::getNumber).map(Integer::longValue).toArray(Long[]::new)) - .properties(properties) - .build(); - } - - - private Long[] toAtomicTextBlockIds(TextBlock textBlock) { - - return textBlock.getAtomicTextBlocks().stream().map(AtomicTextBlock::getId).toArray(Long[]::new); - } - - - private PageData toPageData(Page p) { - - return PageData.builder().rotation(p.getRotation()).height(p.getHeight()).width(p.getWidth()).number(p.getNumber()).build(); - } - - - private static int[] toPrimitiveIntArray(List list) { - - int[] array = new int[list.size()]; - for (int i = 0; i < list.size(); i++) { - array[i] = list.get(i); - } - return array; - } - -} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java index 6abb7e4f..52eec75e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/data/mapper/DocumentGraphMapper.java @@ -65,7 +65,7 @@ public class DocumentGraphMapper { for (DocumentTreeData.EntryData entryData : entries) { boolean terminal = isTerminal(entryData); - List pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList(); + List pages = Arrays.stream(entryData.getPageNumbers()).map(pageNumber -> getPage(pageNumber, context)).toList(); SemanticNode node = switch (entryData.getType()) { case SECTION -> buildSection(context, entryData.getProperties()); @@ -75,12 +75,12 @@ public class DocumentGraphMapper { case FOOTER -> buildFooter(context, terminal); case TABLE -> buildTable(context, entryData.getProperties()); case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal); - case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPages()); + case IMAGE -> buildImage(context, entryData.getProperties(), entryData.getPageNumbers()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); }; if (node.isLeaf()) { - TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node); + TextBlock textBlock = toTextBlock(entryData.getAtomicBlockIds(), context, node); node.setLeafTextBlock(textBlock); } List treeId = Arrays.stream(entryData.getTreeId()).boxed().toList(); @@ -107,7 +107,7 @@ public class DocumentGraphMapper { private static boolean isTerminal(DocumentTreeData.EntryData entryData) { - return entryData.getAtomicBlocks().length > 0; + return entryData.getAtomicBlockIds().length > 0; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java index 93226e60..ff656377 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.AbstractPageBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationDocument; @@ -36,6 +35,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Te import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.experimental.FieldDefaults; @@ -46,11 +46,10 @@ public class DocumentGraphFactory { public Document buildDocumentGraph(ClassificationDocument document) { - TextBlockFactory textBlockFactory = new TextBlockFactory(); Document documentGraph = new Document(); - Context context = new Context(new DocumentTree(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory); + Context context = new Context(documentGraph); - document.getPages().stream().map(DocumentGraphFactory::buildPage).forEach(page -> context.getPages().put(page, new AtomicInteger(1))); + document.getPages().forEach(context::buildAndAddPageWithCounter); document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.getImages().add(image)); addSections(document, context); addHeaderAndFooterToEachPage(document, context); @@ -69,12 +68,6 @@ public class DocumentGraphFactory { } - public static boolean pageBlockIsHeadline(AbstractPageBlock abstractPageBlock) { - - return abstractPageBlock instanceof ClassificationTextBlock && abstractPageBlock.getClassification() != null && abstractPageBlock.getClassification().startsWith("H"); - } - - public void addParagraphOrHeadline(GenericSemanticNode parentNode, ClassificationTextBlock originalTextBlock, Context context, @@ -83,7 +76,7 @@ public class DocumentGraphFactory { Page page = context.getPage(originalTextBlock.getPage()); GenericSemanticNode node; - if (pageBlockIsHeadline(originalTextBlock)) { + if (originalTextBlock.isHeadline()) { node = Headline.builder().documentTree(context.getDocumentTree()).build(); } else { node = Paragraph.builder().documentTree(context.getDocumentTree()).build(); @@ -94,17 +87,9 @@ public class DocumentGraphFactory { List textBlocks = new LinkedList<>(textBlocksToMerge); textBlocks.add(originalTextBlock); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); - - if (node instanceof Headline headline) { - List tocId = context.documentTree.createNewChildEntryAndReturnId(parentNode, NodeType.HEADLINE, node); - headline.setLeafTextBlock(textBlock); - headline.setTreeId(tocId); - } - if (node instanceof Paragraph paragraph) { - List tocId = context.documentTree.createNewChildEntryAndReturnId(parentNode, NodeType.PARAGRAPH, node); - paragraph.setLeafTextBlock(textBlock); - paragraph.setTreeId(tocId); - } + List treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node.getType(), node); + node.setLeafTextBlock(textBlock); + node.setTreeId(treeId); } @@ -178,11 +163,7 @@ public class DocumentGraphFactory { Page page = context.getPage(textBlocks.get(0).getPage()); Header header = Header.builder().documentTree(context.getDocumentTree()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), - header, - context, - 0, - page); + AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page); List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.HEADER, header); header.setTreeId(tocId); header.setLeafTextBlock(textBlock); @@ -214,30 +195,46 @@ public class DocumentGraphFactory { } - private Page buildPage(ClassificationPage p) { - - return Page.builder() - .height((int) p.getPageHeight()) - .width((int) p.getPageWidth()) - .number(p.getPageNumber()) - .rotation(p.getRotation()) - .mainBody(new LinkedList<>()) - .build(); - } - - @Getter @Builder + @AllArgsConstructor @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) - static final class Context { + public static final class Context { DocumentTree documentTree; - Map pages; + Map pages; List
sections; List images; TextBlockFactory textBlockFactory; + public Context(Document document) { + + documentTree = new DocumentTree(document); + pages = new HashMap<>(); + sections = new LinkedList<>(); + images = new LinkedList<>(); + textBlockFactory = new TextBlockFactory(); + } + + + public void buildAndAddPageWithCounter(ClassificationPage classificationPage) { + + Page page = Page.fromClassificationPage(classificationPage); + //this counter counts the TextBlocks per page + //initial value is set to 1, because 0 is reserved for Header + pages.put(page, 1); + } + + + public int getAndIncrementTextBlockNumberOnPage(Page page) { + + Integer textBlockNumberOnPage = pages.get(page); + pages.merge(page, 1, Integer::sum); + return textBlockNumberOnPage; + } + + public Page getPage(int pageIndex) { return pages.keySet() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java index f477eaa5..d6e7ca6e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionDto.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory; import java.awt.geom.Rectangle2D; +import java.util.Collections; import java.util.List; import lombok.AccessLevel; @@ -18,4 +19,15 @@ public class SearchTextWithTextPositionDto { List stringCoordsToPositionCoords; List positions; + + public static SearchTextWithTextPositionDto empty() { + + return SearchTextWithTextPositionDto.builder() + .searchText("") + .lineBreaks(Collections.emptyList()) + .positions(Collections.emptyList()) + .stringCoordsToPositionCoords(Collections.emptyList()) + .build(); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java index fe11f886..3b60e4ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SearchTextWithTextPositionFactory.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory import java.awt.geom.AffineTransform; import java.awt.geom.Rectangle2D; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -19,21 +18,10 @@ public class SearchTextWithTextPositionFactory { public static SearchTextWithTextPositionDto buildSearchTextToTextPositionModel(List sequences) { if (sequences.isEmpty() || sequences.stream().allMatch(sequence -> sequence.getTextPositions().isEmpty())) { - return SearchTextWithTextPositionDto.builder() - .searchText("") - .lineBreaks(Collections.emptyList()) - .positions(Collections.emptyList()) - .stringCoordsToPositionCoords(Collections.emptyList()) - .build(); + return SearchTextWithTextPositionDto.empty(); } - List stringIdxToPositionIdx = new LinkedList<>(); - List lineBreaksStringIdx = new LinkedList<>(); - StringBuilder sb = new StringBuilder(); - - int stringIdx = 0; - int positionIdx = 0; - int lastHyphenIdx = -3; + Context context = new Context(); RedTextPosition currentTextPosition = sequences.get(0).getTextPositions().get(0); RedTextPosition previousTextPosition = RedTextPosition.builder().unicode(" ").position(currentTextPosition.getPosition()).build(); @@ -43,55 +31,72 @@ public class SearchTextWithTextPositionFactory { currentTextPosition = word.getTextPositions().get(i); - if (isLineBreak(currentTextPosition, previousTextPosition)) { - - if (stringIdx - lastHyphenIdx < 3) { - sb.delete(lastHyphenIdx, sb.length()); - stringIdxToPositionIdx = stringIdxToPositionIdx.subList(0, lastHyphenIdx); - stringIdx = lastHyphenIdx; - lastHyphenIdx = -3; - } - lineBreaksStringIdx.add(stringIdx); - } + removeHyphenLinebreaks(currentTextPosition, previousTextPosition, context); if (!isRepeatedWhitespace(currentTextPosition.getUnicode(), previousTextPosition.getUnicode())) { - if (isHyphen(currentTextPosition.getUnicode())) { - lastHyphenIdx = stringIdx; + context.lastHyphenIdx = context.stringIdx; } - sb.append(currentTextPosition.getUnicode()); - - // unicode characters with more than 16 bit encoding have a length > 1 in java strings - for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) { - stringIdxToPositionIdx.add(positionIdx); - } - stringIdx += currentTextPosition.getUnicode().length(); + appendCurrentTextPosition(context, currentTextPosition); } previousTextPosition = currentTextPosition; - ++positionIdx; + ++context.positionIdx; } previousTextPosition = RedTextPosition.builder().unicode(" ").position(previousTextPosition.getPosition()).build(); - sb.append(" "); - stringIdxToPositionIdx.add(positionIdx); - ++stringIdx; + context.sb.append(" "); + context.stringIdxToPositionIdx.add(context.positionIdx); + ++context.stringIdx; } - assert sb.length() == stringIdxToPositionIdx.size(); + assert context.sb.length() == context.stringIdxToPositionIdx.size(); List positions = sequences.stream() .flatMap(sequence -> sequence.getTextPositions().stream().map(textPosition -> mapRedTextPositionToInitialUserSpace(textPosition, sequence))) .toList(); return SearchTextWithTextPositionDto.builder() - .searchText(sb.toString()) - .lineBreaks(lineBreaksStringIdx) - .stringCoordsToPositionCoords(stringIdxToPositionIdx) + .searchText(context.sb.toString()) + .lineBreaks(context.lineBreaksStringIdx) + .stringCoordsToPositionCoords(context.stringIdxToPositionIdx) .positions(positions) .build(); } + private static void appendCurrentTextPosition(Context context, RedTextPosition currentTextPosition) { + + context.sb.append(currentTextPosition.getUnicode()); + + // unicode characters with more than 16-bit encoding have a length > 1 in java strings + for (int j = 0; j < currentTextPosition.getUnicode().length(); j++) { + context.stringIdxToPositionIdx.add(context.positionIdx); + } + context.stringIdx += currentTextPosition.getUnicode().length(); + } + + + private static void removeHyphenLinebreaks(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition, Context context) { + + if (isLineBreak(currentTextPosition, previousTextPosition)) { + + if (lastHyphenDirectlyBeforeLineBreak(context)) { + context.sb.delete(context.lastHyphenIdx, context.sb.length()); + context.stringIdxToPositionIdx = context.stringIdxToPositionIdx.subList(0, context.lastHyphenIdx); + context.stringIdx = context.lastHyphenIdx; + context.lastHyphenIdx = -3; + } + context.lineBreaksStringIdx.add(context.stringIdx); + } + } + + + private static boolean lastHyphenDirectlyBeforeLineBreak(Context context) { + + return context.stringIdx - context.lastHyphenIdx < 3; + } + + private static boolean isLineBreak(RedTextPosition currentTextPosition, RedTextPosition previousTextPosition) { return Objects.equals(currentTextPosition.getUnicode(), "\n") || isDeltaYLargerThanTextHeight(currentTextPosition, previousTextPosition); @@ -156,4 +161,22 @@ public class SearchTextWithTextPositionFactory { return transform.createTransformedShape(rectangle2D).getBounds2D(); } + + private static class Context { + + List stringIdxToPositionIdx = new LinkedList<>(); + List lineBreaksStringIdx = new LinkedList<>(); + StringBuilder sb = new StringBuilder(); + + int stringIdx; + int positionIdx; + // when checking for a hyphen linebreak, we need to check after a linebreak if the last hyphen was less than three symbols away. + // We detect a linebreak as either a "\n" character or if two adjacent symbol's position differ in y-coordinates by at least one character height. + // If there is a hyphen linebreak, the hyphen will be 1 position in front of a "\n" or 2 positions in front of the character which has a lower y-coordinate + // This is why, we need to initialize this to <= -3, otherwise, if the very first symbol is a \n we would detect a hyphen linebreak that isn't there. + // Also, Integer.MIN_VALUE is a bad idea due to potential overflow during arithmetic operations. This is why the default should be -3. + int lastHyphenIdx = -3; + + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java index 4188dbfc..a6053f92 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory; -import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory.pageBlockIsHeadline; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.groupingBy; @@ -36,19 +35,13 @@ public class SectionNodeFactory { context.getSections().add(section); blocksPerPage.keySet().forEach(pageNumber -> addSectionNodeToPageNode(context, section, pageNumber)); - List tocId; - if (parentNode == null) { - tocId = context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.SECTION, section); - } else { - tocId = context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.SECTION, section); - } - section.setTreeId(tocId); + section.setTreeId(getTreeId(parentNode, context, section)); addFirstHeadlineDirectlyToSection(pageBlocks, context, section); if (containsTablesAndTextBlocks(pageBlocks)) { splitPageBlocksIntoSubSections(pageBlocks).forEach(subSectionPageBlocks -> addSection(section, subSectionPageBlocks, emptyList(), context)); } else { - addTablesOrParagraphsOrHeadlinesToSection(pageBlocks, context, section); + addTablesAndParagraphsAndHeadlinesToSection(pageBlocks, context, section); if (listIsTextBlockOnly(pageBlocks)) { section.setExcludesTables(true); } @@ -58,7 +51,17 @@ public class SectionNodeFactory { } - private static boolean listIsTextBlockOnly(List pageBlocks) { + private static List getTreeId(GenericSemanticNode parentNode, DocumentGraphFactory.Context context, Section section) { + + if (parentNode == null) { + return context.getDocumentTree().createNewMainEntryAndReturnId(NodeType.SECTION, section); + } else { + return context.getDocumentTree().createNewChildEntryAndReturnId(parentNode, NodeType.SECTION, section); + } + } + + + private boolean listIsTextBlockOnly(List pageBlocks) { return pageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof ClassificationTextBlock); } @@ -66,14 +69,14 @@ public class SectionNodeFactory { private void addFirstHeadlineDirectlyToSection(List pageBlocks, DocumentGraphFactory.Context context, Section section) { - if (pageBlockIsHeadline(pageBlocks.get(0))) { - addTablesOrParagraphsOrHeadlinesToSection(List.of(pageBlocks.get(0)), context, section); + if (pageBlocks.get(0).isHeadline()) { + addTablesAndParagraphsAndHeadlinesToSection(List.of(pageBlocks.get(0)), context, section); pageBlocks.remove(0); } } - private void addTablesOrParagraphsOrHeadlinesToSection(List pageBlocks, DocumentGraphFactory.Context context, Section section) { + private void addTablesAndParagraphsAndHeadlinesToSection(List pageBlocks, DocumentGraphFactory.Context context, Section section) { Set alreadyMerged = new HashSet<>(); List remainingBlocks = new LinkedList<>(pageBlocks); @@ -100,7 +103,7 @@ public class SectionNodeFactory { } - private static boolean containsTablesAndTextBlocks(List pageBlocks) { + private boolean containsTablesAndTextBlocks(List pageBlocks) { return pageBlocks.stream().anyMatch(pageBlock -> pageBlock instanceof TablePageBlock) && pageBlocks.stream() .anyMatch(pageBlock -> pageBlock instanceof ClassificationTextBlock); @@ -109,8 +112,8 @@ public class SectionNodeFactory { /** * This function splits the list of PageBlocks around TablePageBlocks, such that SubSections can be created, that don't include tables. - * First, the function splits the list into coherent Lists, which include only either type. - * If a Table is directly preceded by a Headline, it is moved to the Table list. + * This is needed so we can execute rules on sections, that do not contain tables. + * See: document structure wiki * * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock * @return List of Lists of AbstractPageBlocks, which include either a single Headline ClassificationTextBlock and a TablePageBlock or only ClassificationTextBlocks. @@ -123,13 +126,13 @@ public class SectionNodeFactory { } - private static void movePrecedingHeadlineToTableList(List> splitList) { + private void movePrecedingHeadlineToTableList(List> splitList) { for (int i = 0; i < splitList.size(); i++) { - if (listIsTableOnly(splitList.get(i)) && i > 0) { + if (listIsTablesOnly(splitList.get(i)) && i > 0) { List previousList = splitList.get(i - 1); AbstractPageBlock lastPageBlockInPreviousList = previousList.get(previousList.size() - 1); - if (pageBlockIsHeadline(lastPageBlockInPreviousList)) { + if (lastPageBlockInPreviousList.isHeadline()) { previousList.remove(i - 1); splitList.get(i).add(0, lastPageBlockInPreviousList); } @@ -138,7 +141,7 @@ public class SectionNodeFactory { } - private static boolean listIsTableOnly(List abstractPageBlocks) { + private boolean listIsTablesOnly(List abstractPageBlocks) { return abstractPageBlocks.stream().allMatch(abstractPageBlock -> abstractPageBlock instanceof TablePageBlock); } @@ -148,7 +151,7 @@ public class SectionNodeFactory { * @param pageBlocks a List of AbstractPageBlocks, which have at least one TablePageBlock and one ClassificationTextBlock * @return List of Lists of AbstractPageBlocks, which are exclusively of type ClassificationTextBlock or TablePageBlock */ - private static List> splitIntoCoherentList(List pageBlocks) { + private List> splitIntoCoherentList(List pageBlocks) { List> splitList = new LinkedList<>(); List currentList = new LinkedList<>(); @@ -169,7 +172,7 @@ public class SectionNodeFactory { } - private static List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) { + private List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) { return pageBlocks.stream() .filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java index b862ca5f..b89f1bc7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java @@ -30,7 +30,7 @@ public class TableNodeFactory { public void addTable(GenericSemanticNode parentNode, List tablesToMerge, DocumentGraphFactory.Context context) { - setPageNumberInCellTextBlocks(tablesToMerge); + setPageNumberInCells(tablesToMerge); Set pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet()); List> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList(); Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build(); @@ -41,22 +41,32 @@ public class TableNodeFactory { table.setTreeId(tocId); addTableCells(mergedRows, table, context); - IfTableHasNoHeadersAssumeFirstRowAreHeaders(table); + ifTableHasNoHeadersAssumeFirstRowAreHeaders(table); } - private static void setPageNumberInCellTextBlocks(List tablesToMerge) { + private void setPageNumberInCells(List tablesToMerge) { + // For some reason I can't figure out, in some table cells, the ClassificationTextBlocks have 0 as page number + // So I am fixing this here, but this should actually be fixed upstream. tablesToMerge.forEach(table -> table.getRows() .stream() .flatMap(Collection::stream) .peek(cell -> cell.setPageNumber(table.getPage())) - .forEach(cell -> cell.getTextBlocks().stream().filter(tb -> tb.getPage() == 0).forEach(tb -> tb.setPage(table.getPage())))); + .forEach(cell -> setPageNumberInTextBlocksWithPageNumberSetTo0(table, cell))); + } + + + private static void setPageNumberInTextBlocksWithPageNumberSetTo0(TablePageBlock table, Cell cell) { + + cell.getTextBlocks().stream()// + .filter(tb -> tb.getPage() == 0)// + .forEach(tb -> tb.setPage(table.getPage())); } @SuppressWarnings("PMD.UnusedPrivateMethod") // PMD actually flags this wrong - private static void addTableToPage(Page page, SemanticNode parentNode, Table table) { + private void addTableToPage(Page page, SemanticNode parentNode, Table table) { if (!page.getMainBody().contains(parentNode)) { parentNode.getPages().add(page); @@ -66,7 +76,7 @@ public class TableNodeFactory { } - private static void IfTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) { + private void ifTableHasNoHeadersAssumeFirstRowAreHeaders(Table table) { if (table.streamHeaders().findAny().isEmpty()) { table.streamRow(0).forEach(tableCellNode -> tableCellNode.setHeader(true)); @@ -124,13 +134,13 @@ public class TableNodeFactory { } - private static boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, Page page) { + private boolean cellAreaIsSmallerThanPageAreaTimesThreshold(Cell cell, Page page) { return cell.getArea() < TABLE_CELL_MERGE_CONTENTS_SIZE_THRESHOLD * page.getHeight() * page.getWidth(); } - private static boolean firstTextBlockIsHeadline(Cell cell) { + private boolean firstTextBlockIsHeadline(Cell cell) { String classification = cell.getTextBlocks().get(0).getClassification(); return classification != null && classification.startsWith("H"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java index 55d31ecb..a1eef095 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java @@ -1,74 +1,53 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory; -import java.util.Collections; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.TextPositionSequence; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; +import lombok.AccessLevel; +import lombok.experimental.FieldDefaults; + +@FieldDefaults(level = AccessLevel.PRIVATE) public class TextBlockFactory { - AtomicInteger stringOffset; - AtomicLong textBlockIdx; - - - public TextBlockFactory() { - - stringOffset = new AtomicInteger(); - textBlockIdx = new AtomicLong(); - } + int stringOffset; + long textBlockIdx; public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) { - Integer numberOnPage = context.getPages().get(page).getAndIncrement(); - return buildAtomicTextBlock(sequences, parent, context, numberOnPage, page); + Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page); + return buildAtomicTextBlock(sequences, parent, numberOnPage, page); } - public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, DocumentGraphFactory.Context context, Integer numberOnPage, Page page) { + public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, Integer numberOnPage, Page page) { SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); - int offset = stringOffset.getAndAdd(searchTextWithTextPositionDto.getSearchText().length()); - - return AtomicTextBlock.builder() - .id(textBlockIdx.getAndIncrement()) - .parent(parent) - .searchText(searchTextWithTextPositionDto.getSearchText()) - .numberOnPage(numberOnPage) - .page(page) - .lineBreaks(searchTextWithTextPositionDto.getLineBreaks()) - .positions(searchTextWithTextPositionDto.getPositions()) - .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords()) - .boundary(new Boundary(offset, offset + searchTextWithTextPositionDto.getSearchText().length())) - .build(); + int offset = stringOffset; + stringOffset += searchTextWithTextPositionDto.getSearchText().length(); + long idx = textBlockIdx; + textBlockIdx++; + return AtomicTextBlock.fromSearchTextWithTextPositionDto(searchTextWithTextPositionDto, parent, offset, idx, numberOnPage, page); } public AtomicTextBlock emptyTextBlock(SemanticNode parent, DocumentGraphFactory.Context context, Page page) { - return emptyTextBlock(parent, context.getPages().get(page).getAndIncrement(), page); + long idx = textBlockIdx; + textBlockIdx++; + return AtomicTextBlock.empty(idx, stringOffset, page, context.getAndIncrementTextBlockNumberOnPage(page), parent); } public AtomicTextBlock emptyTextBlock(SemanticNode parent, Integer numberOnPage, Page page) { - return AtomicTextBlock.builder() - .id(textBlockIdx.getAndIncrement()) - .boundary(new Boundary(stringOffset.get(), stringOffset.get())) - .searchText("") - .lineBreaks(Collections.emptyList()) - .page(page) - .numberOnPage(numberOnPage) - .stringIdxToPositionIdx(Collections.emptyList()) - .positions(Collections.emptyList()) - .parent(parent) - .build(); + long idx = textBlockIdx; + textBlockIdx++; + return AtomicTextBlock.empty(idx, stringOffset, page, numberOnPage, parent); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java index c5e8d73a..53c4021e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java @@ -38,6 +38,13 @@ public class Document implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.DOCUMENT; + } + + public TextBlock buildTextBlock() { return streamTerminalTextBlocksInOrder().collect(new TextBlockCollector()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java index fe2b6d47..d5fe0e2d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java @@ -37,6 +37,13 @@ public class Footer implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.FOOTER; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java index dd04c09b..f6a0060e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java @@ -37,6 +37,13 @@ public class Header implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.HEADER; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java index d56c6ad6..33e51727 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java @@ -37,6 +37,13 @@ public class Headline implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.HEADLINE; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java index ae737b5b..f61662dc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java @@ -55,6 +55,13 @@ public class Image implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.IMAGE; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Page.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Page.java index a03723b0..4e7a0f01 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Page.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Page.java @@ -1,9 +1,11 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Set; +import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.ClassificationPage; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; @@ -44,6 +46,18 @@ public class Page { Set images = new HashSet<>(); + public static Page fromClassificationPage(ClassificationPage classificationPage) { + + return Page.builder() + .height((int) classificationPage.getPageHeight()) + .width((int) classificationPage.getPageWidth()) + .number(classificationPage.getPageNumber()) + .rotation(classificationPage.getRotation()) + .mainBody(new LinkedList<>()) + .build(); + } + + public TextBlock getMainBodyTextBlock() { return mainBody.stream().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java index 4452aef8..6824e9a2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Paragraph.java @@ -35,6 +35,13 @@ public class Paragraph implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.PARAGRAPH; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java index 7bb1fee2..acd79f5e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java @@ -36,6 +36,13 @@ public class Section implements GenericSemanticNode { Set entities = new HashSet<>(); + @Override + public NodeType getType() { + + return NodeType.SECTION; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index 572c97ac..e2f35f2d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -19,6 +19,14 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re public interface SemanticNode { + /** + * Returns the type of this node, such as Section, Paragraph, etc. + * + * @return NodeType of this node + */ + NodeType getType(); + + /** * Searches all Nodes located underneath this Node in the DocumentTree and concatenates their AtomicTextBlocks into a single TextBlockEntity. * So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java index 0000ca59..313beb09 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java @@ -271,6 +271,13 @@ public class Table implements SemanticNode { } + @Override + public NodeType getType() { + + return NodeType.TABLE; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java index 9b3ca02b..3d876a85 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableCell.java @@ -55,6 +55,13 @@ public class TableCell implements GenericSemanticNode { } + @Override + public NodeType getType() { + + return NodeType.TABLE_CELL; + } + + @Override public TextBlock buildTextBlock() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java index f340abcd..1926192f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java @@ -10,9 +10,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import com.google.common.primitives.Ints; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.SearchTextWithTextPositionDto; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; @@ -55,6 +55,43 @@ public class AtomicTextBlock implements TextBlock { } + public static AtomicTextBlock fromSearchTextWithTextPositionDto(SearchTextWithTextPositionDto searchTextWithTextPositionDto, + SemanticNode parent, + int stringOffset, + Long textBlockIdx, + Integer numberOnPage, + Page page) { + + return AtomicTextBlock.builder() + .id(textBlockIdx) + .parent(parent) + .searchText(searchTextWithTextPositionDto.getSearchText()) + .numberOnPage(numberOnPage) + .page(page) + .lineBreaks(searchTextWithTextPositionDto.getLineBreaks()) + .positions(searchTextWithTextPositionDto.getPositions()) + .stringIdxToPositionIdx(searchTextWithTextPositionDto.getStringCoordsToPositionCoords()) + .boundary(new Boundary(stringOffset, stringOffset + searchTextWithTextPositionDto.getSearchText().length())) + .build(); + } + + + public static AtomicTextBlock empty(Long textBlockIdx, int stringOffset, Page page, int numberOnPage, SemanticNode parent) { + + return AtomicTextBlock.builder() + .id(textBlockIdx) + .boundary(new Boundary(stringOffset, stringOffset)) + .searchText("") + .lineBreaks(Collections.emptyList()) + .page(page) + .numberOnPage(numberOnPage) + .stringIdxToPositionIdx(Collections.emptyList()) + .positions(Collections.emptyList()) + .parent(parent) + .build(); + } + + public static AtomicTextBlock fromAtomicTextBlockData(AtomicTextBlockData atomicTextBlockData, AtomicPositionBlockData atomicPositionBlockData, SemanticNode parent, @@ -66,8 +103,8 @@ public class AtomicTextBlock implements TextBlock { .page(page) .boundary(new Boundary(atomicTextBlockData.getStart(), atomicTextBlockData.getEnd())) .searchText(atomicTextBlockData.getSearchText()) - .lineBreaks(Ints.asList(atomicTextBlockData.getLineBreaks())) - .stringIdxToPositionIdx(Ints.asList(atomicPositionBlockData.getStringIdxToPositionIdx())) + .lineBreaks(Arrays.stream(atomicTextBlockData.getLineBreaks()).boxed().toList()) + .stringIdxToPositionIdx(Arrays.stream(atomicPositionBlockData.getStringIdxToPositionIdx()).boxed().toList()) .positions(toRectangle2DList(atomicPositionBlockData.getPositions())) .parent(parent) .build(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java index dd02aafc..5eb83393 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java @@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.t import static java.lang.String.format; import java.awt.geom.Rectangle2D; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -25,6 +26,12 @@ public class ConcatenatedTextBlock implements TextBlock { Boundary boundary; + public static ConcatenatedTextBlock empty() { + + return new ConcatenatedTextBlock(Collections.emptyList()); + } + + public ConcatenatedTextBlock(List atomicTextBlocks) { this.atomicTextBlocks = new LinkedList<>(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java index 89f65b36..119e636f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlockCollector.java @@ -1,6 +1,5 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock; -import java.util.Collections; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.BinaryOperator; @@ -16,7 +15,7 @@ public class TextBlockCollector implements Collector supplier() { - return () -> new ConcatenatedTextBlock(Collections.emptyList()); + return ConcatenatedTextBlock::empty; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 96b6ae92..92931e7b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -33,7 +33,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.mo import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedSectionText; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.model.text.SimplifiedText; import com.iqser.red.service.redaction.v1.server.layoutparsing.classification.service.PdfSegmentationService; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; @@ -109,7 +109,7 @@ public class AnalyzeService { sectionGridCreatorService.createSectionGrid(classifiedDoc, pageCount); log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(document)); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentData.fromDocument(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, toSimplifiedText(document)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java index c1fbc231..8760fe15 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/TextNormalizationUtilities.java @@ -16,4 +16,16 @@ public final class TextNormalizationUtilities { return text.replaceAll("([^\\s\\d\\-]{2,500})[\\-\\u00AD]\\R", "$1"); } + + public static String removeLineBreaks(String text) { + + return text.replaceAll("\n", " "); + } + + + public static String removeRepeatingWhitespaces(String text) { + + return text.replaceAll(" {2}", " "); + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentMappingIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentMappingIntegrationTest.java index c3f1224f..529cdca7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentMappingIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentMappingIntegrationTest.java @@ -10,7 +10,6 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.Ato import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.DocumentTreeData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.PageData; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; @@ -29,7 +28,7 @@ public class DocumentMappingIntegrationTest extends BuildDocumentIntegrationTest String filename = "files/new/crafted document"; Document document = buildGraph(filename); - DocumentData documentData = DocumentDataMapper.toDocumentData(document); + DocumentData documentData = DocumentData.fromDocument(document); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 548983ef..30106e9b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -135,7 +135,6 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi end rule "5: Do not redact Names and Addresses if published information found" - when $section: Section(hasEntitiesOfType("vertebrate"), hasEntitiesOfType("published_information"), @@ -663,8 +662,8 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY" rule "remove Entity of lower rank, when equal boundaries and entityType" salience 32 when - $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary) - $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction) + $higherRank: RedactionEntity($type: type, $entityType: entityType) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then $lowerRank.removeFromGraph(); retract($lowerRank);