Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
* refactored documentGraph.tableOfContents to include document as root
This commit is contained in:
Kilian Schuettler 2023-04-17 12:54:17 +02:00
parent 91e50e495b
commit 8f951ae3b9
40 changed files with 530 additions and 450 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java
View File

@ -4,7 +4,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java
View File

@ -1,20 +1,16 @@
package com.iqser.red.service.redaction.v1.server.document.data; package com.iqser.red.service.redaction.v1.server.document.data;
import static java.lang.String.format;
import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
import javax.management.openmbean.InvalidKeyException;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
@ -25,17 +21,17 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class TableOfContentsData { public class TableOfContentsData {
List<EntryData> entries; EntryData root;
public EntryData get(List<Integer> tocId) { public EntryData get(List<Integer> tocId) {
if (tocId.size() < 1) { if (tocId.isEmpty()) {
throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId)); return root;
} }
EntryData entry = entries.get(tocId.get(0)); EntryData entry = root.subEntries.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) { for (int id : tocId.subList(1, tocId.size())) {
entry = entry.subEntries().get(id); entry = entry.subEntries.get(id);
} }
return entry; return entry;
} }
@ -43,13 +39,7 @@ public class TableOfContentsData {
public Stream<EntryData> streamAllEntries() { public Stream<EntryData> streamAllEntries() {
return entries.stream().flatMap(TableOfContentsData::flatten); return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(TableOfContentsData::flatten);
}
private static List<Integer> getIds(String idsAsString) {
return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
} }
@ -61,12 +51,23 @@ public class TableOfContentsData {
private static Stream<EntryData> flatten(EntryData entry) { private static Stream<EntryData> flatten(EntryData entry) {
return Stream.concat(Stream.of(entry), entry.subEntries().stream().flatMap(TableOfContentsData::flatten)); return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(TableOfContentsData::flatten));
} }
@Builder @Builder
public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map<String, String> properties, List<EntryData> subEntries) { @Getter
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class EntryData {
NodeType type;
int[] tocId;
Long[] atomicBlocks;
Long[] pages;
Map<String, String> properties;
List<EntryData> subEntries;
@Override @Override
public String toString() { public String toString() {

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java
View File

@ -10,6 +10,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
@ -20,13 +21,13 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults; import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data @Data
@Builder @Builder
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE) @FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentGraph implements SemanticNode { public class DocumentGraph implements SemanticNode {
@ -44,7 +45,7 @@ public class DocumentGraph implements SemanticNode {
public List<SectionNode> getMainSections() { public List<SectionNode> getMainSections() {
return tableOfContents.entries.stream().filter(entry -> entry.node() instanceof SectionNode).map(entry -> (SectionNode) entry.node()).collect(Collectors.toList()); return streamChildren().filter(node -> node instanceof SectionNode).map(node -> (SectionNode) node).collect(Collectors.toList());
} }
@ -56,7 +57,7 @@ public class DocumentGraph implements SemanticNode {
public Set<EntityNode> getEntities() { public Set<EntityNode> getEntities() {
return streamAllNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet()); return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
} }
@ -76,14 +77,14 @@ public class DocumentGraph implements SemanticNode {
private Stream<SemanticNode> streamAllNodes() { private Stream<SemanticNode> streamAllNodes() {
return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node); return tableOfContents.streamAllEntriesInOrder().map(TableOfContents.Entry::getNode);
} }
@Override @Override
public String toString() { public String toString() {
return tableOfContents.toString(); return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary();
} }

118
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java
View File

@ -13,30 +13,33 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Data @Data
public class TableOfContents { public class TableOfContents {
List<Entry> entries; private final Entry root;
public TableOfContents() { public TableOfContents(DocumentGraph documentGraph) {
entries = new LinkedList<>(); root = Entry.builder().tocId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(documentGraph).build();
} }
public TextBlock buildTextBlock() { public TextBlock buildTextBlock() {
return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector()); return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
} }
public List<Integer> createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) { public List<Integer> createNewMainEntryAndReturnId(NodeType nodeType, SemanticNode node) {
return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node); return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
} }
@ -44,32 +47,30 @@ public class TableOfContents {
public List<Integer> createNewChildEntryAndReturnId(List<Integer> parentId, NodeType nodeType, SemanticNode node) { public List<Integer> createNewChildEntryAndReturnId(List<Integer> parentId, NodeType nodeType, SemanticNode node) {
List<Integer> newId; if (!entryExists(parentId)) {
if (entryExists(parentId)) { throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId));
Entry parent = getEntryById(parentId);
newId = new LinkedList<>(parentId);
newId.add(parent.children().size());
parent.children().add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
} else {
newId = List.of(entries.size());
entries.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
} }
Entry parent = getEntryById(parentId);
List<Integer> newId = new LinkedList<>(parentId);
newId.add(parent.children.size());
parent.children.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
return newId; return newId;
} }
private boolean entryExists(List<Integer> tocId) { private boolean entryExists(List<Integer> tocId) {
if (tocId.size() < 1) { if (tocId.isEmpty()) {
return false; return root != null;
} }
Entry entry = entries.get(tocId.get(0)); Entry entry = root.children.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) { for (int id : tocId.subList(1, tocId.size())) {
if (id >= entry.children.size() || 0 > id) { if (id >= entry.children.size() || 0 > id) {
return false; return false;
} }
entry = entry.children().get(id); entry = entry.children.get(id);
} }
return true; return true;
} }
@ -77,81 +78,103 @@ public class TableOfContents {
public Entry getParentEntryById(List<Integer> tocId) { public Entry getParentEntryById(List<Integer> tocId) {
List<Integer> parentIds = getParentId(tocId); return getEntryById(getParentId(tocId));
if (parentIds.size() < 1) {
throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId));
}
return getEntryById(parentIds);
} }
public boolean hasParentById(List<Integer> tocId) { public boolean hasParentById(List<Integer> tocId) {
List<Integer> parentId = getParentId(tocId); return entryExists(getParentId(tocId));
return entryExists(parentId);
} }
public Stream<SemanticNode> streamChildren(List<Integer> tocId) { public Stream<SemanticNode> streamChildrenNodes(List<Integer> tocId) {
return getEntryById(tocId).children().stream().map(Entry::node); return getEntryById(tocId).children.stream().map(Entry::getNode);
} }
private static List<Integer> getParentId(List<Integer> tocId) { private static List<Integer> getParentId(List<Integer> tocId) {
if (tocId.isEmpty()) {
throw new UnsupportedOperationException("Root has no parent!");
}
if (tocId.size() < 2) {
return Collections.emptyList();
}
return tocId.subList(0, tocId.size() - 1); return tocId.subList(0, tocId.size() - 1);
} }
public Entry getEntryById(List<Integer> tocId) { public Entry getEntryById(List<Integer> tocId) {
Entry entry = entries.get(tocId.get(0)); if (tocId.isEmpty()) {
return root;
}
Entry entry = root.children.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) { for (int id : tocId.subList(1, tocId.size())) {
entry = entry.children().get(id); entry = entry.children.get(id);
} }
return entry; return entry;
} }
public Stream<Entry> streamEntriesInOrder() { public Stream<Entry> streamMainEntries() {
return entries.stream().flatMap(TableOfContents::flatten); return root.children.stream();
} }
public Stream<Entry> streamSubEntriesInOrder(List<Integer> parentId) { public Stream<Entry> streamAllEntriesInOrder() {
return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten); return Stream.of(root).flatMap(TableOfContents::flatten);
}
public Stream<Entry> streamAllSubEntriesInOrder(List<Integer> parentId) {
return getEntryById(parentId).children.stream().flatMap(TableOfContents::flatten);
} }
@Override @Override
public String toString() { public String toString() {
return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList()); return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList());
}
public String toString(List<Integer> id) {
return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList());
} }
private static Stream<Entry> flatten(Entry entry) { private static Stream<Entry> flatten(Entry entry) {
return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten)); return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(TableOfContents::flatten));
}
public SemanticNode getHighestParentById(List<Integer> tocId) {
if (tocId.isEmpty()) {
return root.node;
}
return root.children.get(tocId.get(0)).node;
} }
@Builder @Builder
public record Entry(List<Integer> tocId, NodeType type, SemanticNode node, List<Entry> children) { @Getter
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public static class Entry {
List<Integer> tocId;
NodeType type;
SemanticNode node;
List<Entry> children;
@Override @Override
public String toString() { public String toString() {
return node().toString(); return node.toString();
} }
@ -161,6 +184,13 @@ public class TableOfContents {
return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode(); return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode();
} }
@Override
public boolean equals(Object o) {
return o instanceof Entry && o.hashCode() == this.hashCode();
}
} }
} }

52
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory; package com.iqser.red.service.redaction.v1.server.document.graph.factory;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.TABLE_CELL;
import static java.lang.String.format; import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toList;
@ -15,10 +10,10 @@ import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set; import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@ -40,7 +35,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@ -56,15 +50,17 @@ public class DocumentGraphFactory {
public DocumentGraph buildDocumentGraph(Document document) { public DocumentGraph buildDocumentGraph(Document document) {
TextBlockFactory textBlockFactory = new TextBlockFactory(); TextBlockFactory textBlockFactory = new TextBlockFactory();
Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory); DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1))); document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image)); document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
addSections(document, context); addSections(document, context);
addHeaderAndFooterToEachPage(document, context); addHeaderAndFooterToEachPage(document, context);
DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build(); documentGraph.setNumberOfPages(context.pages.size());
documentGraph.setPages(context.pages.keySet());
documentGraph.setTableOfContents(context.tableOfContents);
documentGraph.setTextBlock(documentGraph.buildTextBlock()); documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph; return documentGraph;
} }
@ -86,7 +82,7 @@ public class DocumentGraphFactory {
List<Integer> tocId; List<Integer> tocId;
if (parentNode == null) { if (parentNode == null) {
tocId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, sectionNode); tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
} else { } else {
tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode); tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
} }
@ -177,7 +173,7 @@ public class DocumentGraphFactory {
com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock; com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock;
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), TABLE_CELL, tableCellNode); List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
tableCellNode.setTocId(tocId); tableCellNode.setTocId(tocId);
if (cell.getTextBlocks().isEmpty()) { if (cell.getTextBlocks().isEmpty()) {
@ -194,7 +190,7 @@ public class DocumentGraphFactory {
tableCellNode.setTerminal(false); tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
List<TextPositionSequence> sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page); textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock); tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true); tableCellNode.setTerminal(true);
@ -215,7 +211,8 @@ public class DocumentGraphFactory {
private static boolean firstTextBlockIsHeadline(Cell cell) { private static boolean firstTextBlockIsHeadline(Cell cell) {
return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H"); String classification = cell.getTextBlocks().get(0).getClassification();
return classification != null && classification.startsWith("H");
} }
@ -230,7 +227,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(originalTextBlock.getPage(), context); PageNode page = getPage(originalTextBlock.getPage(), context);
SemanticNode node; SemanticNode node;
if (StringUtils.startsWith(originalTextBlock.getClassification(), "H")) { if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build(); node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
} else { } else {
node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build(); node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
@ -240,7 +237,7 @@ public class DocumentGraphFactory {
List<TextBlock> textBlocks = new LinkedList<>(textBlocksToMerge); List<TextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
textBlocks.add(originalTextBlock); textBlocks.add(originalTextBlock);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
if (node instanceof HeadlineNode headlineNode) { if (node instanceof HeadlineNode headlineNode) {
List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node); List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node);
@ -260,7 +257,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(image.getPage(), context); PageNode page = getPage(image.getPage(), context);
ImageNode imageNode = ImageNode.builder() ImageNode imageNode = ImageNode.builder()
.imageType(image.getImageType()) .imageType(image.getImageType())
.position(toRectangle2D(image.getPosition())) .position(RectangleTransformations.toRectangle2D(image.getPosition()))
.transparency(image.isHasTransparency()) .transparency(image.isHasTransparency())
.page(page) .page(page)
.tableOfContents(context.tableOfContents()) .tableOfContents(context.tableOfContents())
@ -308,8 +305,11 @@ public class DocumentGraphFactory {
PageNode page = getPage(textBlocks.get(0).getPage(), context); PageNode page = getPage(textBlocks.get(0).getPage(), context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); footer,
context,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId); footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock); footer.setTerminalTextBlock(textBlock);
page.setFooter(footer); page.setFooter(footer);
@ -320,8 +320,12 @@ public class DocumentGraphFactory {
PageNode page = getPage(textBlocks.get(0).getPage(), context); PageNode page = getPage(textBlocks.get(0).getPage(), context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, context, 0, page); AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); header,
context,
0,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId); header.setTocId(tocId);
header.setTerminalTextBlock(textBlock); header.setTerminalTextBlock(textBlock);
page.setHeader(header); page.setHeader(header);
@ -333,7 +337,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(pageIndex, context); PageNode page = getPage(pageIndex, context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId); footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock); footer.setTerminalTextBlock(textBlock);
page.setFooter(footer); page.setFooter(footer);
@ -345,7 +349,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(pageIndex, context); PageNode page = getPage(pageIndex, context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId); header.setTocId(tocId);
header.setTerminalTextBlock(textBlock); header.setTerminalTextBlock(textBlock);
page.setHeader(header); page.setHeader(header);
@ -370,7 +374,7 @@ public class DocumentGraphFactory {
.stream() .stream()
.filter(page -> page.getNumber() == pageIndex) .filter(page -> page.getNumber() == pageIndex)
.findFirst() .findFirst()
.orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); .orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
} }

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
public enum NodeType { public enum NodeType {
DOCUMENT,
SECTION, SECTION,
HEADLINE, HEADLINE,
PARAGRAPH, PARAGRAPH,

75
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java
View File

@ -1,12 +1,11 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes; package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import static java.util.stream.Collectors.groupingBy;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
@ -20,10 +19,10 @@ public interface SemanticNode {
/** /**
* Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
* So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity * So, for a ClassificationSection all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
* If the Node is Terminal, the TerminalTextBlock will be returned instead. * If the Node is Terminal, the TerminalTextBlock will be returned instead.
* *
* @return TextBlock containing all AtomicTextBlocks that are located under this Node. * @return ClassificationTextBlock containing all AtomicTextBlocks that are located under this Node.
*/ */
TextBlock buildTextBlock(); TextBlock buildTextBlock();
@ -38,7 +37,7 @@ public interface SemanticNode {
/** /**
* Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock.
* *
* @return Set of PageNodes this node appears on. * @return Set of PageNodes this node appears on.
*/ */
@ -49,13 +48,13 @@ public interface SemanticNode {
/** /**
* @return the TableOfContents of the Document this node belongs to * @return the TableOfContents of the ClassificationDocument this node belongs to
*/ */
TableOfContents getTableOfContents(); TableOfContents getTableOfContents();
/** /**
* The id is a List of Integers uniquely identifying this node in the TableOfContents * The id is a List of Integers uniquely identifying this node in the TableOfContents.
* *
* @return the TableOfContents ID * @return the TableOfContents ID
*/ */
@ -63,7 +62,7 @@ public interface SemanticNode {
/** /**
* This should only be used during graph construction * This should only be used during graph construction.
* *
* @param tocId List of Integers * @param tocId List of Integers
*/ */
@ -97,16 +96,26 @@ public interface SemanticNode {
*/ */
default SemanticNode getParent() { default SemanticNode getParent() {
return getTableOfContents().getParentEntryById(getTocId()).node(); return getTableOfContents().getParentEntryById(getTocId()).getNode();
} }
/** /**
* Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. * @return The SemanticNode which is directly underneath the document and also under which this node is.
* if this is the highest child node or the document itself, it returns itself.
*/
default SemanticNode getHighestParent() {
return getTableOfContents().getHighestParentById(getTocId());
}
/**
* Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
* Currently only Sections, Images, and Tables are not terminal. * Currently only Sections, Images, and Tables are not terminal.
* A TableCell might be Terminal depending on its area compared to the page. * A TableCell might be Terminal depending on its area compared to the page.
* *
* @return boolean, indicating if a Node has direct access to a TextBlock * @return boolean, indicating if a Node has direct access to a ClassificationTextBlock
*/ */
default boolean isTerminal() { default boolean isTerminal() {
@ -115,7 +124,7 @@ public interface SemanticNode {
/** /**
* Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
* Currently only Sections and Tables are not terminal. * Currently only Sections and Tables are not terminal.
* *
* @return AtomicTextBlock * @return AtomicTextBlock
@ -150,7 +159,7 @@ public interface SemanticNode {
/** /**
* @return true, if this node's TextBlock is not empty * @return true, if this node's ClassificationTextBlock is not empty
*/ */
default boolean hasText() { default boolean hasText() {
@ -159,8 +168,8 @@ public interface SemanticNode {
/** /**
* @param string A String which the TextBlock might contain * @param string A String which the ClassificationTextBlock might contain
* @return true, if this node's TextBlock contains the string * @return true, if this node's ClassificationTextBlock contains the string
*/ */
default boolean containsString(String string) { default boolean containsString(String string) {
@ -169,8 +178,8 @@ public interface SemanticNode {
/** /**
* @param strings A List of Strings which the TextBlock might contain * @param strings A List of Strings which the ClassificationTextBlock might contain
* @return true, if this node's TextBlock contains any of the strings * @return true, if this node's ClassificationTextBlock contains any of the strings
*/ */
default boolean containsAnyString(List<String> strings) { default boolean containsAnyString(List<String> strings) {
@ -179,34 +188,34 @@ public interface SemanticNode {
/** /**
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity. * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode.
* It sets the fields accordingly and recursively calls this function on all its children. * It sets the fields accordingly and recursively calls this function on all its children.
* *
* @param entity EntityNode, which is being inserted into the graph * @param entityNode EntityNode, which is being inserted into the graph
*/ */
default void addThisToEntityIfIntersects(EntityNode entity) { default void addThisToEntityIfIntersects(EntityNode entityNode) {
TextBlock textBlock = buildTextBlock(); TextBlock textBlock = buildTextBlock();
if (textBlock.getBoundary().intersects(entity.getBoundary())) { if (textBlock.getBoundary().intersects(entityNode.getBoundary())) {
if (textBlock.containsBoundary(entity.getBoundary())) { if (textBlock.containsBoundary(entityNode.getBoundary())) {
entity.setDeepestFullyContainingNode(this); entityNode.setDeepestFullyContainingNode(this);
} }
entity.addIntersectingNode(this); entityNode.addIntersectingNode(this);
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entity)); streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode));
} }
} }
/** /**
* Streams all children located directly underneath this node in the TableOfContents * Streams all children located directly underneath this node in the TableOfContents.
* *
* @return Stream of all children * @return Stream of all children
*/ */
default Stream<SemanticNode> streamChildren() { default Stream<SemanticNode> streamChildren() {
return getTableOfContents().streamChildren(getTocId()); return getTableOfContents().streamChildrenNodes(getTocId());
} }
@ -217,12 +226,12 @@ public interface SemanticNode {
*/ */
default Stream<SemanticNode> streamAllSubNodes() { default Stream<SemanticNode> streamAllSubNodes() {
return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node); return getTableOfContents().streamAllSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::getNode);
} }
/** /**
* @return Boundary of this Node's TextBlock * @return Boundary of this Node's ClassificationTextBlock
*/ */
default Boundary getBoundary() { default Boundary getBoundary() {
@ -232,7 +241,7 @@ public interface SemanticNode {
/** /**
* If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children. * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
* If called on the Document, it will return the cropbox of each page * If called on the ClassificationDocument, it will return the cropbox of each page
* *
* @return Rectangle2D fully encapsulating this Node for each page. * @return Rectangle2D fully encapsulating this Node for each page.
*/ */
@ -248,7 +257,7 @@ public interface SemanticNode {
/** /**
* TODO this does not yet work for sections spanning multiple columns * TODO this does not yet work for sections spanning multiple columns.
* *
* @param bBoxPerPage initial empty BoundingBox * @param bBoxPerPage initial empty BoundingBox
* @return The union of the BoundingBoxes of all children * @return The union of the BoundingBoxes of all children
@ -264,11 +273,11 @@ public interface SemanticNode {
/** /**
* @param bBoxPerPage initial empty BoundingBox * @param bBoxPerPage initial empty BoundingBox
* @return The union of all BoundingBoxes of the TextBlock of this node * @return The union of all BoundingBoxes of the ClassificationTextBlock of this node
*/ */
private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) { private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage)); Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs))); atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
return bBoxPerPage; return bBoxPerPage;
} }

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java
View File

@ -5,8 +5,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
@ -21,7 +19,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@Service import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentDataMapper { public class DocumentDataMapper {
public DocumentData toDocumentData(DocumentGraph documentGraph) { public DocumentData toDocumentData(DocumentGraph documentGraph) {
@ -29,16 +29,16 @@ public class DocumentDataMapper {
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder() List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct() .distinct()
.map(this::toAtomicTextBlockData) .map(DocumentDataMapper::toAtomicTextBlockData)
.toList(); .toList();
List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder() List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct() .distinct()
.map(this::toAtomicPositionBlockData) .map(DocumentDataMapper::toAtomicPositionBlockData)
.toList(); .toList();
List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList(); List<PageData> pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList();
TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents()); TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
return DocumentData.builder() return DocumentData.builder()
.atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0])) .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
@ -51,7 +51,7 @@ public class DocumentDataMapper {
private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) { private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList()); return new TableOfContentsData(toEntryData(tableOfContents.getRoot()));
} }
@ -59,25 +59,25 @@ public class DocumentDataMapper {
Long[] atomicTextBlocks; Long[] atomicTextBlocks;
if (entry.node().isTerminal()) { if (entry.getNode().isTerminal()) {
atomicTextBlocks = toAtomicTextBlockIds(entry.node().getTerminalTextBlock()); atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getTerminalTextBlock());
} else { } else {
atomicTextBlocks = new Long[]{}; atomicTextBlocks = new Long[]{};
} }
Map<String, String> properties = switch (entry.type()) { Map<String, String> properties = switch (entry.getType()) {
case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.node()); case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.node()); case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.node()); case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
default -> new HashMap<>(); default -> new HashMap<>();
}; };
return TableOfContentsData.EntryData.builder() return TableOfContentsData.EntryData.builder()
.tocId(toPrimitiveIntArray(entry.tocId())) .tocId(toPrimitiveIntArray(entry.getTocId()))
.subEntries(entry.children().stream().map(this::toEntryData).toList()) .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
.type(entry.type()) .type(entry.getType())
.atomicBlocks(atomicTextBlocks) .atomicBlocks(atomicTextBlocks)
.pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new)) .pages(entry.getNode().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
.properties(properties) .properties(properties)
.build(); .build();
} }

73
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java
View File

@ -1,10 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.mapper; package com.iqser.red.service.redaction.v1.server.document.mapper;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageProperties; import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableCellProperties; import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableProperties;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import java.awt.geom.Rectangle2D; import java.awt.geom.Rectangle2D;
import java.util.Arrays; import java.util.Arrays;
@ -12,9 +9,7 @@ import java.util.HashSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException;
import org.apache.commons.lang3.NotImplementedException;
import org.springframework.stereotype.Service;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
@ -29,7 +24,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
@ -39,29 +33,30 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
@Service import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentGraphMapper { public class DocumentGraphMapper {
public DocumentGraph toDocumentGraph(DocumentData documentData) { public DocumentGraph toDocumentGraph(DocumentData documentData) {
DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(documentData, Context context = new Context(documentData,
new TableOfContents(), new TableOfContents(documentGraph),
new LinkedList<>(), new LinkedList<>(),
new LinkedList<>(), new LinkedList<>(),
Arrays.stream(documentData.getAtomicTextBlocks()).toList(), Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
Arrays.stream(documentData.getAtomicPositionBlocks()).toList()); Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList()); context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context)); context.tableOfContents.getRoot().getChildren().addAll(buildEntries(documentData.getTableOfContents().getRoot().getSubEntries(), context));
documentGraph.setTableOfContents(context.tableOfContents);
documentGraph.setPages(new HashSet<>(context.pages));
documentGraph.setNumberOfPages(documentData.getPages().length);
DocumentGraph documentGraph = DocumentGraph.builder()
.numberOfPages(documentData.getPages().length)
.pages(new HashSet<>(context.pages))
.tableOfContents(context.tableOfContents)
.build();
documentGraph.setTextBlock(documentGraph.buildTextBlock()); documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph; return documentGraph;
} }
@ -73,35 +68,35 @@ public class DocumentGraphMapper {
for (TableOfContentsData.EntryData entryData : entries) { for (TableOfContentsData.EntryData entryData : entries) {
boolean terminal = isTerminal(entryData); boolean terminal = isTerminal(entryData);
List<PageNode> pages = Arrays.stream(entryData.pages()).map(pageNumber -> getPage(pageNumber, context)).toList(); List<PageNode> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
SemanticNode node = switch (entryData.type()) { SemanticNode node = switch (entryData.getType()) {
case SECTION -> buildSection(context); case SECTION -> buildSection(context);
case PARAGRAPH -> buildParagraph(context, terminal); case PARAGRAPH -> buildParagraph(context, terminal);
case HEADLINE -> buildHeadline(context, terminal); case HEADLINE -> buildHeadline(context, terminal);
case HEADER -> buildHeader(context, terminal); case HEADER -> buildHeader(context, terminal);
case FOOTER -> buildFooter(context, terminal); case FOOTER -> buildFooter(context, terminal);
case TABLE -> buildTable(context, entryData.properties()); case TABLE -> buildTable(context, entryData.getProperties());
case TABLE_CELL -> buildTableCell(context, entryData.properties(), terminal); case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
case IMAGE -> buildImage(context, entryData.properties()); case IMAGE -> buildImage(context, entryData.getProperties());
default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type()); default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
}; };
if (node.isTerminal()) { if (node.isTerminal()) {
TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node); TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node);
node.setTerminalTextBlock(textBlock); node.setTerminalTextBlock(textBlock);
} }
List<Integer> tocId = Arrays.stream(entryData.tocId()).boxed().toList(); List<Integer> tocId = Arrays.stream(entryData.getTocId()).boxed().toList();
node.setTocId(tocId); node.setTocId(tocId);
if (entryData.type() == NodeType.HEADER) { if (entryData.getType() == HEADER) {
pages.forEach(page -> page.setHeader((HeaderNode) node)); pages.forEach(page -> page.setHeader((HeaderNode) node));
} else if (entryData.type() == NodeType.FOOTER) { } else if (entryData.getType() == FOOTER) {
pages.forEach(page -> page.setFooter((FooterNode) node)); pages.forEach(page -> page.setFooter((FooterNode) node));
} else { } else {
pages.forEach(page -> page.getMainBody().add(node)); pages.forEach(page -> page.getMainBody().add(node));
} }
newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.type()).children(buildEntries(entryData.subEntries(), context)).node(node).build()); newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build());
} }
return newEntries; return newEntries;
} }
@ -115,14 +110,14 @@ public class DocumentGraphMapper {
private static boolean isTerminal(TableOfContentsData.EntryData entryData) { private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
return entryData.atomicBlocks().length > 0; return entryData.getAtomicBlocks().length > 0;
} }
private ImageNode buildImage(Context context, Map<String, String> properties) { private ImageNode buildImage(Context context, Map<String, String> properties) {
var builder = ImageNode.builder(); var builder = ImageNode.builder();
parseImageProperties(properties, builder); PropertiesMapper.parseImageProperties(properties, builder);
return builder.tableOfContents(context.tableOfContents()).build(); return builder.tableOfContents(context.tableOfContents()).build();
} }
@ -130,7 +125,7 @@ public class DocumentGraphMapper {
private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) { private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder(); TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder();
parseTableCellProperties(properties, builder); PropertiesMapper.parseTableCellProperties(properties, builder);
return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build(); return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build();
} }
@ -138,7 +133,7 @@ public class DocumentGraphMapper {
private TableNode buildTable(Context context, Map<String, String> properties) { private TableNode buildTable(Context context, Map<String, String> properties) {
TableNode.TableNodeBuilder builder = TableNode.builder(); TableNode.TableNodeBuilder builder = TableNode.builder();
parseTableProperties(properties, builder); PropertiesMapper.parseTableProperties(properties, builder);
return TableNode.builder().tableOfContents(context.tableOfContents()).build(); return TableNode.builder().tableOfContents(context.tableOfContents()).build();
} }
@ -171,8 +166,8 @@ public class DocumentGraphMapper {
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds) return Arrays.stream(atomicTextBlockIds)
.map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)),
context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)), context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
parent, parent,
context)) context))
.collect(new TextBlockCollector()); .collect(new TextBlockCollector());
@ -210,14 +205,14 @@ public class DocumentGraphMapper {
private PageNode getPage(Long pageIndex, Context context) { private PageNode getPage(Long pageIndex, Context context) {
return context.pages.stream() return context.pages.stream()
.filter(page -> page.getNumber() == toIntExact(pageIndex)) .filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
.findFirst() .findFirst()
.orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
} }
record Context( record Context(
DocumentData documentData, DocumentData layoutParsingModel,
TableOfContents tableOfContents, TableOfContents tableOfContents,
List<PageNode> pages, List<PageNode> pages,
List<SectionNode> sections, List<SectionNode> sections,

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -5,8 +5,10 @@ import static com.iqser.red.service.redaction.v1.server.redaction.utils.Separato
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -19,9 +21,8 @@ import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@ -122,14 +123,18 @@ public class EntityCreationService {
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
throw new IllegalArgumentException("Provided entities can not be merged!"); throw new IllegalArgumentException("Provided entities can not be merged!");
} }
if (entitiesToMerge.size() < 2) {
throw new IllegalArgumentException("more than 2 entities are required to merge!");
}
EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType); EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType);
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction)); mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction));
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
if (mergedEntity.isRedaction()) { EntityNode entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(EntityNode::getMatchedRule)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse("")); mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1)); mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse("")); mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
}
addEntityToGraph(mergedEntity, node.getTableOfContents()); addEntityToGraph(mergedEntity, node.getTableOfContents());
return mergedEntity; return mergedEntity;
} }
@ -144,12 +149,10 @@ public class EntityCreationService {
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
try { try {
SemanticNode containingNode = tableOfContents.getEntries() SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
.stream()
.map(TableOfContents.Entry::node)
.filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary())) .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
.findFirst() .findFirst()
.orElseThrow(() -> new NotFoundException("No containing Node found!")); .orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
containingNode.addThisToEntityIfIntersects(entity); containingNode.addThisToEntityIfIntersects(entity);
@ -159,9 +162,7 @@ public class EntityCreationService {
addToPages(entity); addToPages(entity);
addToNodeEntitySets(entity); addToNodeEntitySets(entity);
} catch (NotFoundException e) { } catch (NoSuchElementException e) {
entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock());
log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary());
entity.removeFromGraph(); entity.removeFromGraph();
} }
} }

100
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -26,12 +26,13 @@ import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import lombok.Builder; import lombok.Builder;
import lombok.Data; import lombok.Data;
@ -207,6 +208,7 @@ public class Section {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) { public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) {
@ -214,6 +216,7 @@ public class Section {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue())); return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue()));
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@WhenCondition @WhenCondition
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
@ -571,12 +574,12 @@ public class Section {
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern, public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group, @Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType, @Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber, @Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason, @Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false); redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
} }
@ -665,7 +668,8 @@ public class Section {
legalBasis, legalBasis,
true, true,
skipRemoveEntitiesContainedInLarger, skipRemoveEntitiesContainedInLarger,
sortedResult, false); sortedResult,
false);
} }
@ -697,11 +701,11 @@ public class Section {
legalBasis, legalBasis,
true, true,
skipRemoveEntitiesContainedInLarger, skipRemoveEntitiesContainedInLarger,
sortedResult, ignoreTables); sortedResult,
ignoreTables);
} }
@ThenAction @ThenAction
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start, public void redactBetween(@Argument(ArgumentType.STRING) String start,
@ -729,7 +733,8 @@ public class Section {
legalBasis, legalBasis,
true, true,
skipRemoveEntitiesContainedInLarger, skipRemoveEntitiesContainedInLarger,
sortedResult, false); sortedResult,
false);
} }
@ -772,7 +777,8 @@ public class Section {
legalBasis, legalBasis,
true, true,
skipRemoveEntitiesContainedInLarger, skipRemoveEntitiesContainedInLarger,
sortedResult, false); sortedResult,
false);
} }
} }
@ -787,19 +793,7 @@ public class Section {
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactBetween(start, redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, null, false, false, false, false);
stop,
false,
false,
asType,
ruleNumber,
redactEverywhere,
false,
reason,
null,
false,
false,
false, false);
} }
@ -813,19 +807,7 @@ public class Section {
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason) { @Argument(ArgumentType.STRING) String reason) {
redactBetween(start, redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false, false, false);
stop,
false,
false,
asType,
ruleNumber,
redactEverywhere,
excludeHeadLine,
reason,
null,
false,
false,
false, false);
} }
@ -1293,14 +1275,14 @@ public class Section {
public Set<Entity> findEntities(String value, public Set<Entity> findEntities(String value,
String asType, String asType,
boolean caseInsensitive, boolean caseInsensitive,
boolean redacted, boolean redacted,
int ruleNumber, int ruleNumber,
String reason, String reason,
String legalBasis, String legalBasis,
Engine engine, Engine engine,
boolean asRecommendation) { boolean asRecommendation) {
String text = caseInsensitive ? searchText.toLowerCase() : searchText; String text = caseInsensitive ? searchText.toLowerCase() : searchText;
Set<Entity> found = EntitySearchUtils.findEntities(text, Set<Entity> found = EntitySearchUtils.findEntities(text,
@ -1445,7 +1427,8 @@ public class Section {
if (StringUtils.isNotBlank(stringOffset.getValue())) { if (StringUtils.isNotBlank(stringOffset.getValue())) {
var trimmedOffsetString = stringOffset.trim(); var trimmedOffsetString = stringOffset.trim();
Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream() Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
.filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet()); .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
.collect(Collectors.toSet());
found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger)); found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
@ -1459,7 +1442,15 @@ public class Section {
} }
private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) { private void redactByRegExWithNewlines(String pattern,
boolean patternCaseInsensitive,
int group,
String asType,
int ruleNumber,
String reason,
String legalBasis,
boolean redaction,
boolean skipRemoveEntitiesContainedInLarger) {
Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive);
@ -1476,7 +1467,15 @@ public class Section {
} }
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) { private void redactByRegEx(String pattern,
boolean patternCaseInsensitive,
int group,
String asType,
int ruleNumber,
String reason,
String legalBasis,
boolean redaction,
boolean skipRemoveEntitiesContainedInLarger) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -1524,8 +1523,7 @@ public class Section {
boolean sortedResult, boolean sortedResult,
boolean ignoreTables) { boolean ignoreTables) {
if (isInTable && ignoreTables) {
if(isInTable && ignoreTables){
return; return;
} }

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
@ -12,4 +14,5 @@ public class TenantDictionary {
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>(); private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>(); private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
} }

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import static java.lang.String.format; import static java.lang.String.format;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.Set;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import java.util.Set;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class DictionaryIncrement { public class DictionaryIncrement {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java
View File

@ -1,16 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.io.Serializable; import java.io.Serializable;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data @Data
@AllArgsConstructor @AllArgsConstructor
public class DictionaryModel implements Serializable { public class DictionaryModel implements Serializable {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import lombok.Data;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import lombok.Data;
@Data @Data
public class DictionaryRepresentation { public class DictionaryRepresentation {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils; package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -22,14 +22,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import feign.FeignException; import feign.FeignException;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -33,9 +33,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section; import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -104,7 +104,7 @@ public class DroolsExecutionService {
kieSession.setGlobal("dictionary", dictionary); kieSession.setGlobal("dictionary", dictionary);
document.getEntities().forEach(kieSession::insert); document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
document.getPages().forEach(kieSession::insert); document.getPages().forEach(kieSession::insert);
fileAttributes.forEach(kieSession::insert); fileAttributes.forEach(kieSession::insert);

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
@ -18,12 +19,11 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
@ -37,6 +37,7 @@ public class ManualRedactionSurroundingTextService {
private final RedactionStorageService redactionStorageService; private final RedactionStorageService redactionStorageService;
private final RedactionServiceSettings redactionServiceSettings; private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_surroundingTextAnalysis") @Timed("redactmanager_surroundingTextAnalysis")
public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) { public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@ -87,9 +88,18 @@ public class ManualRedactionSurroundingTextService {
} }
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) { if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); SurroundingWordsService.addSurroundingText(Set.of(correctEntity),
sectionText.getSearchableText(),
null,
sectionText.getCellStarts(),
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
} else { } else {
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); SurroundingWordsService.addSurroundingText(Set.of(correctEntity),
sectionText.getSearchableText(),
null,
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
} }
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter()); return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java
View File

@ -3,9 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
@ -15,7 +15,6 @@ import lombok.extern.slf4j.Slf4j;
@UtilityClass @UtilityClass
public class SurroundingWordsService { public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingText") @Timed("redactmanager_addSurroundingText")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) { public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
@ -38,7 +37,12 @@ public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingTextTables") @Timed("redactmanager_addSurroundingTextTables")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) { public void addSurroundingText(Set<Entity> entities,
SearchableText searchableText,
Dictionary dictionary,
List<Integer> cellstarts,
int surroundingWordsOffsetWindow,
int numberOfSurroundingWords) {
if (entities.isEmpty()) { if (entities.isEmpty()) {
return; return;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -39,10 +39,10 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException; import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService; import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
@ -86,8 +86,6 @@ public class AnalyzeService {
ImportedRedactionService importedRedactionService; ImportedRedactionService importedRedactionService;
SectionFinder sectionFinder; SectionFinder sectionFinder;
DocumentGraphFactory documentGraphFactory; DocumentGraphFactory documentGraphFactory;
DocumentDataMapper documentDataMapper;
DocumentGraphMapper documentGraphMapper;
FunctionTimerValues redactmanagerAnalyzePagewiseValues; FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@ -133,7 +131,7 @@ public class AnalyzeService {
sectionText.getSectionAreas()))); sectionText.getSectionAreas())));
log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
@ -154,7 +152,7 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
// not yet ready for reanalysis // not yet ready for reanalysis
if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) { if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
@ -208,7 +206,7 @@ public class AnalyzeService {
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
NerEntities nerEntities; NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) { if (redactionServiceSettings.isNerServiceEnabled()) {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -21,11 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import io.micrometer.core.annotation.Timed; import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel; import lombok.AccessLevel;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java
View File

@ -11,12 +11,12 @@ import org.springframework.stereotype.Component;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -13,9 +13,9 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import lombok.AccessLevel; import lombok.AccessLevel;

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -13,13 +13,14 @@ import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.experimental.UtilityClass; import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java
View File

@ -34,7 +34,7 @@ public class PdfDraw {
public static void drawDocumentGraph(PDDocument document, DocumentGraph documentGraph) { public static void drawDocumentGraph(PDDocument document, DocumentGraph documentGraph) {
documentGraph.getTableOfContents().streamEntriesInOrder().forEach(entry -> drawNode(document, entry)); documentGraph.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> drawNode(document, entry));
} }
@ -132,7 +132,8 @@ public class PdfDraw {
private static Options buildStandardOptionsForNodes(TableOfContents.Entry entry) { private static Options buildStandardOptionsForNodes(TableOfContents.Entry entry) {
return Options.builder().stroke(true).strokeColor(switch (entry.type()) { return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
case DOCUMENT -> Color.LIGHT_GRAY;
case HEADER, FOOTER -> Color.GREEN; case HEADER, FOOTER -> Color.GREEN;
case PARAGRAPH -> Color.BLUE; case PARAGRAPH -> Color.BLUE;
case HEADLINE -> Color.RED; case HEADLINE -> Color.RED;
@ -146,9 +147,9 @@ public class PdfDraw {
private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, TableOfContents.Entry entry, Options options) { private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, TableOfContents.Entry entry, Options options) {
Map<PageNode, Rectangle2D> rectanglesPerPage = entry.node().getBBox(); Map<PageNode, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
rectanglesPerPage.forEach((page, rectangle2D) -> { rectanglesPerPage.forEach((page, rectangle2D) -> {
if (entry.type() == NodeType.SECTION) { if (entry.getType() == NodeType.SECTION) {
rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10); rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
} }
drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options); drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
@ -159,7 +160,7 @@ public class PdfDraw {
private static String buildString(TableOfContents.Entry entry) { private static String buildString(TableOfContents.Entry entry) {
return entry.node().getNumberOnPage() + ": " + entry.tocId() + ": " + entry.type().toString(); return entry.getNode().getNumberOnPage() + ": " + entry.getTocId() + ": " + entry.getType().toString();
} }
} }

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java
View File

@ -65,10 +65,10 @@ import lombok.SneakyThrows;
@Import(AbstractTestWithDictionaries.TestConfiguration.class) @Import(AbstractTestWithDictionaries.TestConfiguration.class)
public class AbstractTestWithDictionaries { public class AbstractTestWithDictionaries {
protected static final String RULES = loadFromClassPath("drools/rules.drl"); protected static final String RULES = loadFromClassPath("drools/testRules.drl");
protected static final String RULES_PATH = "drools/rules.drl"; protected static final String RULES_PATH = "drools/rules.drl";
protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl"; protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
protected static final String MERGE_ENTITY_RULES_PATH = "drools/merge_entity_rules.drl"; protected static final String MERGE_ENTITY_RULES_PATH = "drools/testRules.drl";
private static final String VERTEBRATE = "vertebrate"; private static final String VERTEBRATE = "vertebrate";
private static final String ADDRESS = "CBI_address"; private static final String ADDRESS = "CBI_address";
private static final String AUTHOR = "CBI_author"; private static final String AUTHOR = "CBI_author";
@ -518,7 +518,7 @@ public class AbstractTestWithDictionaries {
KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8")); kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8"));
kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8")); //kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8"));
KieRepository kieRepository = kieServices.getRepository(); KieRepository kieRepository = kieServices.getRepository();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -30,7 +30,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -28,12 +28,12 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw; import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
@ -98,7 +98,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
long serializationStart = System.currentTimeMillis(); long serializationStart = System.currentTimeMillis();
document.getEntities().forEach(kieSession::insert); document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
document.getPages().forEach(kieSession::insert); document.getPages().forEach(kieSession::insert);
System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart); System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart);
@ -140,7 +140,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
durationMillis = ((float) (System.currentTimeMillis() - start)); durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).node(); SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).getNode();
start = System.currentTimeMillis(); start = System.currentTimeMillis();
for (int i = 0; i < n; i++) { for (int i = 0; i < n; i++) {
paragraph.buildTextBlock(); paragraph.buildTextBlock();

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.server; package com.iqser.red.service.redaction.v1.server;
import static java.io.File.createTempFile;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@ -15,7 +17,6 @@ import org.springframework.core.io.InputStreamResource;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory; import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService; import com.iqser.red.storage.commons.service.StorageService;
@ -62,7 +63,7 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows @SneakyThrows
public <T> void storeJSONObject(String tenantId, String objectId, T any) { public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = FileUtils.createTempFile("test", ".tmp"); File tempFile = createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any); getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile); dataMap.put(objectId, tempFile);
} }
@ -101,7 +102,7 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows @SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) { public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = FileUtils.createTempFile("test", ".tmp"); File tempFile = createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) { try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream); IOUtils.copy(stream, fileOutputStream);

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java
View File

@ -143,6 +143,7 @@ public class AnnotationService {
annotation.setTitlePopup(redactionLogEntry.getId()); annotation.setTitlePopup(redactionLogEntry.getId());
annotation.setAnnotationName(redactionLogEntry.getId()); annotation.setAnnotationName(redactionLogEntry.getId());
annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE)); annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
annotation.setNoRotate(false);
annotations.add(annotation); annotations.add(annotation);
if (redactionLogEntry.getComments() != null) { if (redactionLogEntry.getComments() != null) {

61
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java Normal file
View File

@ -0,0 +1,61 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows;
public class BuildDocumentGraphTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired
private ImageService imageService;
@Test
public void buildMetolachlor() {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
assertEquals(221, documentGraph.getPages().size());
assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
}
@SneakyThrows
protected DocumentGraph buildGraph(String filename) {
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
} else {
prepareStorage(filename + ".pdf");
}
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<PdfImage>> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
return documentGraphFactory.buildDocumentGraph(classifiedDoc);
}
}
}

143
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionTest.java
View File

@ -4,18 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.wildfly.common.Assert.assertTrue; import static org.wildfly.common.Assert.assertTrue;
import java.io.InputStream;
import java.util.List; import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@ -26,25 +20,37 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows; public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
public class DocumentGraphTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired @Autowired
private EntityCreationService entityCreationService; private EntityCreationService entityCreationService;
@Autowired
private ImageService imageService; @Test
public void assertCollectAllEntitiesWorks() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
createAndInsertEntity(documentGraph, "Clarissa");
createAndInsertEntity(documentGraph, "Lastname");
createAndInsertEntity(documentGraph, "David Ksenia");
createAndInsertEntity(documentGraph, "Michael N.");
createAndInsertEntity(documentGraph, "Page-Footer");
createAndInsertEntity(documentGraph, "CTL/with dictionary entry 1234 with Slash");
assertEquals(6, documentGraph.getEntities().size());
}
private EntityNode createAndInsertEntity(DocumentGraph documentGraph, String searchTerm) {
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
return entityNode;
}
@Test @Test
@ -52,11 +58,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document"); DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "Clarissa"; String searchTerm = "Clarissa";
int start = documentGraph.getTextBlock().indexOf(searchTerm); EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("Expand to Hint ", entityNode.getTextBefore());
assertEquals("’s Donut ←", entityNode.getTextAfter()); assertEquals("’s Donut ←", entityNode.getTextAfter());
@ -67,7 +69,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@ -76,11 +78,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document"); DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "Rule 39:"; String searchTerm = "Rule 39:";
int start = documentGraph.getTextBlock().indexOf(searchTerm); EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextBefore());
assertEquals(" Purity Hint", entityNode.getTextAfter()); assertEquals(" Purity Hint", entityNode.getTextAfter());
@ -90,7 +88,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@ -99,11 +97,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document"); DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "1998"; String searchTerm = "1998";
int start = documentGraph.getTextBlock().indexOf(searchTerm); EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextBefore());
assertEquals("", entityNode.getTextAfter()); assertEquals("", entityNode.getTextAfter());
@ -113,7 +107,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(15, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(15, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@ -134,9 +128,9 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document"); DocumentGraph documentGraph = buildGraph("files/new/crafted document");
TableNode table = (TableNode) documentGraph.getTableOfContents()// TableNode table = (TableNode) documentGraph.getTableOfContents()//
.streamEntriesInOrder()// .streamAllEntriesInOrder()//
.filter(entry -> entry.type().equals(NodeType.TABLE))// .filter(entry -> entry.getType().equals(NodeType.TABLE))//
.map(TableOfContents.Entry::node)// .map(TableOfContents.Entry::getNode)//
.findFirst().orElseThrow(); .findFirst().orElseThrow();
assertEquals(5, table.getNumberOfCols()); assertEquals(5, table.getNumberOfCols());
assertEquals(4, table.getNumberOfRows()); assertEquals(4, table.getNumberOfRows());
@ -164,10 +158,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
TableNode table = (TableNode) documentGraph.getTableOfContents() TableNode table = (TableNode) documentGraph.getTableOfContents()
.streamEntriesInOrder() .streamAllEntriesInOrder()
.filter(entry -> entry.node().getPages().stream().anyMatch(page -> page.getNumber() == 22)) .filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22))
.filter(entry -> entry.type().equals(NodeType.TABLE)) .filter(entry -> entry.getType().equals(NodeType.TABLE))
.map(TableOfContents.Entry::node) .map(TableOfContents.Entry::getNode)
.findFirst() .findFirst()
.orElseThrow(); .orElseThrow();
assertEquals(5, table.getNumberOfCols()); assertEquals(5, table.getNumberOfCols());
@ -185,11 +179,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
String searchTerm = "Cucurbit"; String searchTerm = "Cucurbit";
int start = documentGraph.getTextBlock().indexOf(searchTerm); EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
assertEquals(", Group 9;", entityNode.getTextAfter()); assertEquals(", Group 9;", entityNode.getTextAfter());
@ -197,9 +187,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(searchTerm, entityNode.getValue()); assertEquals(searchTerm, entityNode.getValue());
assertEquals(2, entityNode.getIntersectingNodes().size()); assertEquals(2, entityNode.getIntersectingNodes().size());
assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10));
assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@ -214,7 +205,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
assertEquals("2.6.1 Summary of ", entityNode.getTextBefore()); assertEquals("2.6.1 Summary of ", entityNode.getTextBefore());
assertEquals(" and excretion in", entityNode.getTextAfter()); assertEquals(" and excretion in", entityNode.getTextAfter());
@ -222,10 +214,11 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
assertEquals(searchTerm, entityNode.getValue()); assertEquals(searchTerm, entityNode.getValue());
assertEquals(2, entityNode.getIntersectingNodes().size()); assertEquals(2, entityNode.getIntersectingNodes().size());
assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertEquals(4, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33));
assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@ -234,11 +227,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
String searchTerm = "N-deacetylation product"; String searchTerm = "N-deacetylation product";
int start = documentGraph.getTextBlock().indexOf(searchTerm); EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
assertEquals(" of metabolite of", entityNode.getTextAfter()); assertEquals(" of metabolite of", entityNode.getTextAfter());
@ -246,43 +235,23 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(3, entityNode.getIntersectingNodes().size()); assertEquals(3, entityNode.getIntersectingNodes().size());
assertEquals("2.7.2 Summary of metabolism, distribution and expression of residues in plants, poultry, lactating ruminants, pigs and fish ", assertEquals("2.7.2 Summary of metabolism, distribution and expression of residues in plants, poultry, lactating ruminants, pigs and fish ",
entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54));
// TODO: the number below is odd, due the cell (1, 2) containing 4 AtomicTextBlocks, even though it is an image. This should be fixed during structure analysis. assertEquals(26, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
// In general, numberOnPage needs some more thought.
assertEquals(25, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode()); assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
} }
@SneakyThrows private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, EntityNode entityNode) {
protected DocumentGraph buildGraph(String filename) {
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
} else {
prepareStorage(filename + ".pdf");
}
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<PdfImage>> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
return documentGraphFactory.buildDocumentGraph(classifiedDoc);
}
}
private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, int start, EntityNode entityNode) {
List<Integer> paragraphStart = entityNode.getIntersectingNodes().stream()// List<Integer> paragraphStart = entityNode.getIntersectingNodes().stream()//
.map(SemanticNode::buildTextBlock)// .map(SemanticNode::buildTextBlock)//
.map(textBlock -> textBlock.indexOf(searchTerm))// .map(textBlock -> textBlock.indexOf(searchTerm))//
.toList(); .toList();
paragraphStart.forEach(nodeStart -> assertEquals(start, nodeStart)); paragraphStart.forEach(nodeStart -> assertEquals(entityNode.getBoundary().start(), nodeStart));
} }
@ -293,15 +262,17 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
assertEquals(entityNode.getValue(), searchTerm); assertEquals(entityNode.getValue(), searchTerm);
assertTrue(pageNode.getEntities().contains(entityNode)); assertTrue(pageNode.getEntities().contains(entityNode));
assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode))); assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode)));
assertTrue(entityNode.getPages().contains(pageNode)); assertTrue(entityNode.getPages().contains(pageNode));
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode))); assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode)));
} }
} }

35
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java
View File

@ -1,37 +1,19 @@
package com.iqser.red.service.redaction.v1.server.document.graph; package com.iqser.red.service.redaction.v1.server.document.graph;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows; import lombok.SneakyThrows;
public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired
private DocumentDataMapper documentDataMapper;
@Autowired
private DocumentGraphMapper documentGraphMapper;
@Test @Test
@SneakyThrows @SneakyThrows
@ -39,12 +21,9 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
String filename = "files/new/crafted document"; String filename = "files/new/crafted document";
prepareStorage(filename + ".pdf"); DocumentGraph document = buildGraph(filename);
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); DocumentData documentData = DocumentDataMapper.toDocumentData(document);
var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
DocumentData documentData = documentDataMapper.toDocumentData(document);
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks());
@ -57,7 +36,13 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
AtomicPositionBlockData[].class); AtomicPositionBlockData[].class);
TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class); TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class);
DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData); DocumentData documentData2 = DocumentData.builder()
.pages(pageData)
.tableOfContents(tableOfContentsData)
.atomicTextBlocks(atomicTextBlockData)
.atomicPositionBlocks(atomicPositionBlockData)
.build();
DocumentGraph newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2);
assert document.toString().equals(newDocumentGraph.toString()); assert document.toString().equals(newDocumentGraph.toString());
assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString()); assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java
View File

@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
import lombok.SneakyThrows; import lombok.SneakyThrows;
public class DocumentGraphVisualizationTest extends DocumentGraphTest { public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
@Test @Test
@SneakyThrows @SneakyThrows

45
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -18,18 +18,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
import java.util.Set import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
global DocumentGraph document global DocumentGraph document
global EntityCreationService entityCreationService global EntityCreationService entityCreationService
global Dictionary dictionary global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes" query "getFileAttributes"
$fileAttribute: FileAttribute() $fileAttribute: FileAttribute()
end end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type" rule "merge intersecting Entities of same type"
salience 100 salience 100
@ -50,10 +53,10 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
when when
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE) $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) $entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then then
entity.removeFromGraph(); $entity.removeFromGraph();
retract(entity); retract($entity);
end end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
@ -89,6 +92,8 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
retract($lowerRank); retract($lowerRank);
end end
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search" rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999 salience -999
@ -108,11 +113,11 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true no-loop true
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then then
$entity.setRedaction(true); $entity.setRedaction(true);
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); setFields($entity, 1, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity) update($entity)
end end
@ -120,11 +125,11 @@ rule "2: Redact CBI Authors (Vertebrate study)"
no-loop true no-loop true
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then then
$entity.setRedaction(true); $entity.setRedaction(true);
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); setFields($entity, 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity) update($entity)
end end
@ -132,7 +137,7 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
no-loop true no-loop true
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then then
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
@ -156,9 +161,9 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then then
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); EntityNode falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE); setFields(falsePositive, 5, "Genitive Author", null, Engine.RULE);
insert(entity); insert(falsePositive);
end end
@ -189,7 +194,7 @@ rule "7: Add CBI_author with \"et al.\" Regex"
rule "8: Add recommendation for Addresses in Test Organism sections" rule "8: Add recommendation for Addresses in Test Organism sections"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") //FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then then
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
@ -228,7 +233,7 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study
no-loop true no-loop true
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then then
$entity.setRedaction(true); $entity.setRedaction(true);
@ -253,7 +258,7 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
rule "12: Redact Emails by RegEx (Non vertebrate study)" rule "12: Redact Emails by RegEx (Non vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@")) $section: SectionNode(containsString("@"))
then then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
@ -277,7 +282,7 @@ rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:", $string: String() from List.of("Contact point:",
"Contact:", "Contact:",
"Alternative contact:", "Alternative contact:",
@ -289,7 +294,7 @@ rule "14: Redact line after contact information (Non vertebrate study)"
"Telephone number:", "Telephone number:",
"Telephone No:", "Telephone No:",
"Telephone:", "Telephone:",
"Phone No:", "Phone No.",
"Phone:", "Phone:",
"Fax number:", "Fax number:",
"Fax:", "Fax:",
@ -324,7 +329,7 @@ rule "15: Redact line after contact information (Vertebrate study)"
"Telephone number:", "Telephone number:",
"Telephone No:", "Telephone No:",
"Telephone:", "Telephone:",
"Phone No:", "Phone No.",
"Phone:", "Phone:",
"Fax number:", "Fax number:",
"Fax:", "Fax:",
@ -345,7 +350,7 @@ rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then then
Set<EntityNode> entities = new HashSet<>(); Set<EntityNode> entities = new HashSet<>();