Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
* refactored documentGraph.tableOfContents to include document as root
This commit is contained in:
Kilian Schuettler 2023-04-17 12:54:17 +02:00
parent 9d8e544d0c
commit aa86e94f70
40 changed files with 530 additions and 450 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java
View File

@ -4,7 +4,7 @@ import java.util.ArrayList;
import java.util.List;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import lombok.Data;
import lombok.NoArgsConstructor;

39
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java
View File

@ -1,20 +1,16 @@
package com.iqser.red.service.redaction.v1.server.document.data;
import static java.lang.String.format;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import javax.management.openmbean.InvalidKeyException;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
@ -25,17 +21,17 @@ import lombok.experimental.FieldDefaults;
@FieldDefaults(level = AccessLevel.PRIVATE)
public class TableOfContentsData {
List<EntryData> entries;
EntryData root;
public EntryData get(List<Integer> tocId) {
if (tocId.size() < 1) {
throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId));
if (tocId.isEmpty()) {
return root;
}
EntryData entry = entries.get(tocId.get(0));
EntryData entry = root.subEntries.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) {
entry = entry.subEntries().get(id);
entry = entry.subEntries.get(id);
}
return entry;
}
@ -43,13 +39,7 @@ public class TableOfContentsData {
public Stream<EntryData> streamAllEntries() {
return entries.stream().flatMap(TableOfContentsData::flatten);
}
private static List<Integer> getIds(String idsAsString) {
return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList();
return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(TableOfContentsData::flatten);
}
@ -61,12 +51,23 @@ public class TableOfContentsData {
private static Stream<EntryData> flatten(EntryData entry) {
return Stream.concat(Stream.of(entry), entry.subEntries().stream().flatMap(TableOfContentsData::flatten));
return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(TableOfContentsData::flatten));
}
@Builder
public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map<String, String> properties, List<EntryData> subEntries) {
@Getter
@AllArgsConstructor
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public static class EntryData {
NodeType type;
int[] tocId;
Long[] atomicBlocks;
Long[] pages;
Map<String, String> properties;
List<EntryData> subEntries;
@Override
public String toString() {

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java
View File

@ -10,6 +10,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
@ -20,13 +21,13 @@ import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
public class DocumentGraph implements SemanticNode {
@ -44,7 +45,7 @@ public class DocumentGraph implements SemanticNode {
public List<SectionNode> getMainSections() {
return tableOfContents.entries.stream().filter(entry -> entry.node() instanceof SectionNode).map(entry -> (SectionNode) entry.node()).collect(Collectors.toList());
return streamChildren().filter(node -> node instanceof SectionNode).map(node -> (SectionNode) node).collect(Collectors.toList());
}
@ -56,7 +57,7 @@ public class DocumentGraph implements SemanticNode {
public Set<EntityNode> getEntities() {
return streamAllNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet());
}
@ -76,14 +77,14 @@ public class DocumentGraph implements SemanticNode {
private Stream<SemanticNode> streamAllNodes() {
return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node);
return tableOfContents.streamAllEntriesInOrder().map(TableOfContents.Entry::getNode);
}
@Override
public String toString() {
return tableOfContents.toString();
return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary();
}

118
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java
View File

@ -13,30 +13,33 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.experimental.FieldDefaults;
@Data
public class TableOfContents {
List<Entry> entries;
private final Entry root;
public TableOfContents() {
public TableOfContents(DocumentGraph documentGraph) {
entries = new LinkedList<>();
root = Entry.builder().tocId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(documentGraph).build();
}
public TextBlock buildTextBlock() {
return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector());
}
public List<Integer> createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) {
public List<Integer> createNewMainEntryAndReturnId(NodeType nodeType, SemanticNode node) {
return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node);
}
@ -44,32 +47,30 @@ public class TableOfContents {
public List<Integer> createNewChildEntryAndReturnId(List<Integer> parentId, NodeType nodeType, SemanticNode node) {
List<Integer> newId;
if (entryExists(parentId)) {
Entry parent = getEntryById(parentId);
newId = new LinkedList<>(parentId);
newId.add(parent.children().size());
parent.children().add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
} else {
newId = List.of(entries.size());
entries.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
if (!entryExists(parentId)) {
throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId));
}
Entry parent = getEntryById(parentId);
List<Integer> newId = new LinkedList<>(parentId);
newId.add(parent.children.size());
parent.children.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build());
return newId;
}
private boolean entryExists(List<Integer> tocId) {
if (tocId.size() < 1) {
return false;
if (tocId.isEmpty()) {
return root != null;
}
Entry entry = entries.get(tocId.get(0));
Entry entry = root.children.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) {
if (id >= entry.children.size() || 0 > id) {
return false;
}
entry = entry.children().get(id);
entry = entry.children.get(id);
}
return true;
}
@ -77,81 +78,103 @@ public class TableOfContents {
public Entry getParentEntryById(List<Integer> tocId) {
List<Integer> parentIds = getParentId(tocId);
if (parentIds.size() < 1) {
throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId));
}
return getEntryById(parentIds);
return getEntryById(getParentId(tocId));
}
public boolean hasParentById(List<Integer> tocId) {
List<Integer> parentId = getParentId(tocId);
return entryExists(parentId);
return entryExists(getParentId(tocId));
}
public Stream<SemanticNode> streamChildren(List<Integer> tocId) {
public Stream<SemanticNode> streamChildrenNodes(List<Integer> tocId) {
return getEntryById(tocId).children().stream().map(Entry::node);
return getEntryById(tocId).children.stream().map(Entry::getNode);
}
private static List<Integer> getParentId(List<Integer> tocId) {
if (tocId.isEmpty()) {
throw new UnsupportedOperationException("Root has no parent!");
}
if (tocId.size() < 2) {
return Collections.emptyList();
}
return tocId.subList(0, tocId.size() - 1);
}
public Entry getEntryById(List<Integer> tocId) {
Entry entry = entries.get(tocId.get(0));
if (tocId.isEmpty()) {
return root;
}
Entry entry = root.children.get(tocId.get(0));
for (int id : tocId.subList(1, tocId.size())) {
entry = entry.children().get(id);
entry = entry.children.get(id);
}
return entry;
}
public Stream<Entry> streamEntriesInOrder() {
public Stream<Entry> streamMainEntries() {
return entries.stream().flatMap(TableOfContents::flatten);
return root.children.stream();
}
public Stream<Entry> streamSubEntriesInOrder(List<Integer> parentId) {
public Stream<Entry> streamAllEntriesInOrder() {
return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten);
return Stream.of(root).flatMap(TableOfContents::flatten);
}
public Stream<Entry> streamAllSubEntriesInOrder(List<Integer> parentId) {
return getEntryById(parentId).children.stream().flatMap(TableOfContents::flatten);
}
@Override
public String toString() {
return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList());
}
public String toString(List<Integer> id) {
return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList());
return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList());
}
private static Stream<Entry> flatten(Entry entry) {
return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten));
return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(TableOfContents::flatten));
}
public SemanticNode getHighestParentById(List<Integer> tocId) {
if (tocId.isEmpty()) {
return root.node;
}
return root.children.get(tocId.get(0)).node;
}
@Builder
public record Entry(List<Integer> tocId, NodeType type, SemanticNode node, List<Entry> children) {
@Getter
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true)
public static class Entry {
List<Integer> tocId;
NodeType type;
SemanticNode node;
List<Entry> children;
@Override
public String toString() {
return node().toString();
return node.toString();
}
@ -161,6 +184,13 @@ public class TableOfContents {
return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode();
}
@Override
public boolean equals(Object o) {
return o instanceof Entry && o.hashCode() == this.hashCode();
}
}
}

52
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java
View File

@ -1,10 +1,5 @@
package com.iqser.red.service.redaction.v1.server.document.graph.factory;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D;
import static com.iqser.red.service.redaction.v1.server.document.graph.factory.TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.TABLE_CELL;
import static java.lang.String.format;
import static java.util.stream.Collectors.groupingBy;
import static java.util.stream.Collectors.toList;
@ -15,10 +10,10 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
@ -40,7 +35,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
@ -56,15 +50,17 @@ public class DocumentGraphFactory {
public DocumentGraph buildDocumentGraph(Document document) {
TextBlockFactory textBlockFactory = new TextBlockFactory();
Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory);
document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1)));
document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image));
addSections(document, context);
addHeaderAndFooterToEachPage(document, context);
DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build();
documentGraph.setNumberOfPages(context.pages.size());
documentGraph.setPages(context.pages.keySet());
documentGraph.setTableOfContents(context.tableOfContents);
documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph;
}
@ -86,7 +82,7 @@ public class DocumentGraphFactory {
List<Integer> tocId;
if (parentNode == null) {
tocId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, sectionNode);
tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode);
} else {
tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode);
}
@ -177,7 +173,7 @@ public class DocumentGraphFactory {
com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock;
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), TABLE_CELL, tableCellNode);
List<Integer> tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode);
tableCellNode.setTocId(tocId);
if (cell.getTextBlocks().isEmpty()) {
@ -194,7 +190,7 @@ public class DocumentGraphFactory {
tableCellNode.setTerminal(false);
} else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) {
List<TextPositionSequence> sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
List<TextPositionSequence> sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks());
textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page);
tableCellNode.setTerminalTextBlock(textBlock);
tableCellNode.setTerminal(true);
@ -215,7 +211,8 @@ public class DocumentGraphFactory {
private static boolean firstTextBlockIsHeadline(Cell cell) {
return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H");
String classification = cell.getTextBlocks().get(0).getClassification();
return classification != null && classification.startsWith("H");
}
@ -230,7 +227,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(originalTextBlock.getPage(), context);
SemanticNode node;
if (StringUtils.startsWith(originalTextBlock.getClassification(), "H")) {
if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) {
node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build();
} else {
node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build();
@ -240,7 +237,7 @@ public class DocumentGraphFactory {
List<TextBlock> textBlocks = new LinkedList<>(textBlocksToMerge);
textBlocks.add(originalTextBlock);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page);
if (node instanceof HeadlineNode headlineNode) {
List<Integer> tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node);
@ -260,7 +257,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(image.getPage(), context);
ImageNode imageNode = ImageNode.builder()
.imageType(image.getImageType())
.position(toRectangle2D(image.getPosition()))
.position(RectangleTransformations.toRectangle2D(image.getPosition()))
.transparency(image.isHasTransparency())
.page(page)
.tableOfContents(context.tableOfContents())
@ -308,8 +305,11 @@ public class DocumentGraphFactory {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
footer,
context,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock);
page.setFooter(footer);
@ -320,8 +320,12 @@ public class DocumentGraphFactory {
PageNode page = getPage(textBlocks.get(0).getPage(), context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, context, 0, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks),
header,
context,
0,
page);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId);
header.setTerminalTextBlock(textBlock);
page.setHeader(header);
@ -333,7 +337,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(pageIndex, context);
FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer);
footer.setTocId(tocId);
footer.setTerminalTextBlock(textBlock);
page.setFooter(footer);
@ -345,7 +349,7 @@ public class DocumentGraphFactory {
PageNode page = getPage(pageIndex, context);
HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build();
AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page);
List<Integer> tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header);
List<Integer> tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header);
header.setTocId(tocId);
header.setTerminalTextBlock(textBlock);
page.setHeader(header);
@ -370,7 +374,7 @@ public class DocumentGraphFactory {
.stream()
.filter(page -> page.getNumber() == pageIndex)
.findFirst()
.orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex)));
.orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex)));
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java
View File

@ -1,6 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
public enum NodeType {
DOCUMENT,
SECTION,
HEADLINE,
PARAGRAPH,

75
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java
View File

@ -1,12 +1,11 @@
package com.iqser.red.service.redaction.v1.server.document.graph.nodes;
import static java.util.stream.Collectors.groupingBy;
import java.awt.geom.Rectangle2D;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
@ -20,10 +19,10 @@ public interface SemanticNode {
/**
* Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity.
* So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
* So, for a ClassificationSection all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity
* If the Node is Terminal, the TerminalTextBlock will be returned instead.
*
* @return TextBlock containing all AtomicTextBlocks that are located under this Node.
* @return ClassificationTextBlock containing all AtomicTextBlocks that are located under this Node.
*/
TextBlock buildTextBlock();
@ -38,7 +37,7 @@ public interface SemanticNode {
/**
* Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock
* Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock.
*
* @return Set of PageNodes this node appears on.
*/
@ -49,13 +48,13 @@ public interface SemanticNode {
/**
* @return the TableOfContents of the Document this node belongs to
* @return the TableOfContents of the ClassificationDocument this node belongs to
*/
TableOfContents getTableOfContents();
/**
* The id is a List of Integers uniquely identifying this node in the TableOfContents
* The id is a List of Integers uniquely identifying this node in the TableOfContents.
*
* @return the TableOfContents ID
*/
@ -63,7 +62,7 @@ public interface SemanticNode {
/**
* This should only be used during graph construction
* This should only be used during graph construction.
*
* @param tocId List of Integers
*/
@ -97,16 +96,26 @@ public interface SemanticNode {
*/
default SemanticNode getParent() {
return getTableOfContents().getParentEntryById(getTocId()).node();
return getTableOfContents().getParentEntryById(getTocId()).getNode();
}
/**
* Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
* @return The SemanticNode which is directly underneath the document and also under which this node is.
* if this is the highest child node or the document itself, it returns itself.
*/
default SemanticNode getHighestParent() {
return getTableOfContents().getHighestParentById(getTocId());
}
/**
* Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
* Currently only Sections, Images, and Tables are not terminal.
* A TableCell might be Terminal depending on its area compared to the page.
*
* @return boolean, indicating if a Node has direct access to a TextBlock
* @return boolean, indicating if a Node has direct access to a ClassificationTextBlock
*/
default boolean isTerminal() {
@ -115,7 +124,7 @@ public interface SemanticNode {
/**
* Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden.
* Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden.
* Currently only Sections and Tables are not terminal.
*
* @return AtomicTextBlock
@ -150,7 +159,7 @@ public interface SemanticNode {
/**
* @return true, if this node's TextBlock is not empty
* @return true, if this node's ClassificationTextBlock is not empty
*/
default boolean hasText() {
@ -159,8 +168,8 @@ public interface SemanticNode {
/**
* @param string A String which the TextBlock might contain
* @return true, if this node's TextBlock contains the string
* @param string A String which the ClassificationTextBlock might contain
* @return true, if this node's ClassificationTextBlock contains the string
*/
default boolean containsString(String string) {
@ -169,8 +178,8 @@ public interface SemanticNode {
/**
* @param strings A List of Strings which the TextBlock might contain
* @return true, if this node's TextBlock contains any of the strings
* @param strings A List of Strings which the ClassificationTextBlock might contain
* @return true, if this node's ClassificationTextBlock contains any of the strings
*/
default boolean containsAnyString(List<String> strings) {
@ -179,34 +188,34 @@ public interface SemanticNode {
/**
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity.
* This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode.
* It sets the fields accordingly and recursively calls this function on all its children.
*
* @param entity EntityNode, which is being inserted into the graph
* @param entityNode EntityNode, which is being inserted into the graph
*/
default void addThisToEntityIfIntersects(EntityNode entity) {
default void addThisToEntityIfIntersects(EntityNode entityNode) {
TextBlock textBlock = buildTextBlock();
if (textBlock.getBoundary().intersects(entity.getBoundary())) {
if (textBlock.getBoundary().intersects(entityNode.getBoundary())) {
if (textBlock.containsBoundary(entity.getBoundary())) {
entity.setDeepestFullyContainingNode(this);
if (textBlock.containsBoundary(entityNode.getBoundary())) {
entityNode.setDeepestFullyContainingNode(this);
}
entity.addIntersectingNode(this);
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entity));
entityNode.addIntersectingNode(this);
streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode));
}
}
/**
* Streams all children located directly underneath this node in the TableOfContents
* Streams all children located directly underneath this node in the TableOfContents.
*
* @return Stream of all children
*/
default Stream<SemanticNode> streamChildren() {
return getTableOfContents().streamChildren(getTocId());
return getTableOfContents().streamChildrenNodes(getTocId());
}
@ -217,12 +226,12 @@ public interface SemanticNode {
*/
default Stream<SemanticNode> streamAllSubNodes() {
return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node);
return getTableOfContents().streamAllSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::getNode);
}
/**
* @return Boundary of this Node's TextBlock
* @return Boundary of this Node's ClassificationTextBlock
*/
default Boundary getBoundary() {
@ -232,7 +241,7 @@ public interface SemanticNode {
/**
* If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
* If called on the Document, it will return the cropbox of each page
* If called on the ClassificationDocument, it will return the cropbox of each page
*
* @return Rectangle2D fully encapsulating this Node for each page.
*/
@ -248,7 +257,7 @@ public interface SemanticNode {
/**
* TODO this does not yet work for sections spanning multiple columns
* TODO this does not yet work for sections spanning multiple columns.
*
* @param bBoxPerPage initial empty BoundingBox
* @return The union of the BoundingBoxes of all children
@ -264,11 +273,11 @@ public interface SemanticNode {
/**
* @param bBoxPerPage initial empty BoundingBox
* @return The union of all BoundingBoxes of the TextBlock of this node
* @return The union of all BoundingBoxes of the ClassificationTextBlock of this node
*/
private Map<PageNode, Rectangle2D> getBBoxFromTerminalTextBlock(Map<PageNode, Rectangle2D> bBoxPerPage) {
Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage));
Map<PageNode, List<AtomicTextBlock>> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage));
atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs)));
return bBoxPerPage;
}

34
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java
View File

@ -5,8 +5,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
@ -21,7 +19,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
@Service
import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentDataMapper {
public DocumentData toDocumentData(DocumentGraph documentGraph) {
@ -29,16 +29,16 @@ public class DocumentDataMapper {
List<AtomicTextBlockData> atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicTextBlockData)
.map(DocumentDataMapper::toAtomicTextBlockData)
.toList();
List<AtomicPositionBlockData> atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder()
.flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream())
.distinct()
.map(this::toAtomicPositionBlockData)
.map(DocumentDataMapper::toAtomicPositionBlockData)
.toList();
List<PageData> pageData = documentGraph.getPages().stream().map(this::toPageData).toList();
List<PageData> pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList();
TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents());
return DocumentData.builder()
.atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0]))
@ -51,7 +51,7 @@ public class DocumentDataMapper {
private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) {
return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList());
return new TableOfContentsData(toEntryData(tableOfContents.getRoot()));
}
@ -59,25 +59,25 @@ public class DocumentDataMapper {
Long[] atomicTextBlocks;
if (entry.node().isTerminal()) {
atomicTextBlocks = toAtomicTextBlockIds(entry.node().getTerminalTextBlock());
if (entry.getNode().isTerminal()) {
atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getTerminalTextBlock());
} else {
atomicTextBlocks = new Long[]{};
}
Map<String, String> properties = switch (entry.type()) {
case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.node());
case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.node());
case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.node());
Map<String, String> properties = switch (entry.getType()) {
case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode());
case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode());
case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode());
default -> new HashMap<>();
};
return TableOfContentsData.EntryData.builder()
.tocId(toPrimitiveIntArray(entry.tocId()))
.subEntries(entry.children().stream().map(this::toEntryData).toList())
.type(entry.type())
.tocId(toPrimitiveIntArray(entry.getTocId()))
.subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList())
.type(entry.getType())
.atomicBlocks(atomicTextBlocks)
.pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
.pages(entry.getNode().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new))
.properties(properties)
.build();
}

73
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java
View File

@ -1,10 +1,7 @@
package com.iqser.red.service.redaction.v1.server.document.mapper;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageProperties;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableCellProperties;
import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableProperties;
import static java.lang.Math.toIntExact;
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER;
import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER;
import java.awt.geom.Rectangle2D;
import java.util.Arrays;
@ -12,9 +9,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.NotImplementedException;
import org.springframework.stereotype.Service;
import java.util.NoSuchElementException;
import com.google.common.primitives.Ints;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
@ -29,7 +24,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
@ -39,29 +33,30 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
@Service
import lombok.experimental.UtilityClass;
@UtilityClass
public class DocumentGraphMapper {
public DocumentGraph toDocumentGraph(DocumentData documentData) {
DocumentGraph documentGraph = new DocumentGraph();
Context context = new Context(documentData,
new TableOfContents(),
new TableOfContents(documentGraph),
new LinkedList<>(),
new LinkedList<>(),
Arrays.stream(documentData.getAtomicTextBlocks()).toList(),
Arrays.stream(documentData.getAtomicPositionBlocks()).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList());
context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList());
context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context));
context.tableOfContents.getRoot().getChildren().addAll(buildEntries(documentData.getTableOfContents().getRoot().getSubEntries(), context));
documentGraph.setTableOfContents(context.tableOfContents);
documentGraph.setPages(new HashSet<>(context.pages));
documentGraph.setNumberOfPages(documentData.getPages().length);
DocumentGraph documentGraph = DocumentGraph.builder()
.numberOfPages(documentData.getPages().length)
.pages(new HashSet<>(context.pages))
.tableOfContents(context.tableOfContents)
.build();
documentGraph.setTextBlock(documentGraph.buildTextBlock());
return documentGraph;
}
@ -73,35 +68,35 @@ public class DocumentGraphMapper {
for (TableOfContentsData.EntryData entryData : entries) {
boolean terminal = isTerminal(entryData);
List<PageNode> pages = Arrays.stream(entryData.pages()).map(pageNumber -> getPage(pageNumber, context)).toList();
List<PageNode> pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList();
SemanticNode node = switch (entryData.type()) {
SemanticNode node = switch (entryData.getType()) {
case SECTION -> buildSection(context);
case PARAGRAPH -> buildParagraph(context, terminal);
case HEADLINE -> buildHeadline(context, terminal);
case HEADER -> buildHeader(context, terminal);
case FOOTER -> buildFooter(context, terminal);
case TABLE -> buildTable(context, entryData.properties());
case TABLE_CELL -> buildTableCell(context, entryData.properties(), terminal);
case IMAGE -> buildImage(context, entryData.properties());
default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type());
case TABLE -> buildTable(context, entryData.getProperties());
case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal);
case IMAGE -> buildImage(context, entryData.getProperties());
default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType());
};
if (node.isTerminal()) {
TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node);
TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node);
node.setTerminalTextBlock(textBlock);
}
List<Integer> tocId = Arrays.stream(entryData.tocId()).boxed().toList();
List<Integer> tocId = Arrays.stream(entryData.getTocId()).boxed().toList();
node.setTocId(tocId);
if (entryData.type() == NodeType.HEADER) {
if (entryData.getType() == HEADER) {
pages.forEach(page -> page.setHeader((HeaderNode) node));
} else if (entryData.type() == NodeType.FOOTER) {
} else if (entryData.getType() == FOOTER) {
pages.forEach(page -> page.setFooter((FooterNode) node));
} else {
pages.forEach(page -> page.getMainBody().add(node));
}
newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.type()).children(buildEntries(entryData.subEntries(), context)).node(node).build());
newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build());
}
return newEntries;
}
@ -115,14 +110,14 @@ public class DocumentGraphMapper {
private static boolean isTerminal(TableOfContentsData.EntryData entryData) {
return entryData.atomicBlocks().length > 0;
return entryData.getAtomicBlocks().length > 0;
}
private ImageNode buildImage(Context context, Map<String, String> properties) {
var builder = ImageNode.builder();
parseImageProperties(properties, builder);
PropertiesMapper.parseImageProperties(properties, builder);
return builder.tableOfContents(context.tableOfContents()).build();
}
@ -130,7 +125,7 @@ public class DocumentGraphMapper {
private TableCellNode buildTableCell(Context context, Map<String, String> properties, boolean terminal) {
TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder();
parseTableCellProperties(properties, builder);
PropertiesMapper.parseTableCellProperties(properties, builder);
return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build();
}
@ -138,7 +133,7 @@ public class DocumentGraphMapper {
private TableNode buildTable(Context context, Map<String, String> properties) {
TableNode.TableNodeBuilder builder = TableNode.builder();
parseTableProperties(properties, builder);
PropertiesMapper.parseTableProperties(properties, builder);
return TableNode.builder().tableOfContents(context.tableOfContents()).build();
}
@ -171,8 +166,8 @@ public class DocumentGraphMapper {
private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) {
return Arrays.stream(atomicTextBlockIds)
.map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)),
context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)),
.map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)),
context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)),
parent,
context))
.collect(new TextBlockCollector());
@ -210,14 +205,14 @@ public class DocumentGraphMapper {
private PageNode getPage(Long pageIndex, Context context) {
return context.pages.stream()
.filter(page -> page.getNumber() == toIntExact(pageIndex))
.filter(page -> page.getNumber() == Math.toIntExact(pageIndex))
.findFirst()
.orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex)));
.orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex)));
}
record Context(
DocumentData documentData,
DocumentData layoutParsingModel,
TableOfContents tableOfContents,
List<PageNode> pages,
List<SectionNode> sections,

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -5,8 +5,10 @@ import static com.iqser.red.service.redaction.v1.server.redaction.utils.Separato
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@ -19,9 +21,8 @@ import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -122,14 +123,18 @@ public class EntityCreationService {
if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) {
throw new IllegalArgumentException("Provided entities can not be merged!");
}
if (entitiesToMerge.size() < 2) {
throw new IllegalArgumentException("more than 2 entities are required to merge!");
}
EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType);
mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction));
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
if (mergedEntity.isRedaction()) {
mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse(""));
mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1));
mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse(""));
}
EntityNode entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(EntityNode::getMatchedRule)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule());
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
addEntityToGraph(mergedEntity, node.getTableOfContents());
return mergedEntity;
}
@ -144,12 +149,10 @@ public class EntityCreationService {
public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) {
try {
SemanticNode containingNode = tableOfContents.getEntries()
.stream()
.map(TableOfContents.Entry::node)
SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList())
.filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary()))
.findFirst()
.orElseThrow(() -> new NotFoundException("No containing Node found!"));
.orElseThrow(() -> new NoSuchElementException("No containing Node found!"));
containingNode.addThisToEntityIfIntersects(entity);
@ -159,9 +162,7 @@ public class EntityCreationService {
addToPages(entity);
addToNodeEntitySets(entity);
} catch (NotFoundException e) {
entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock());
log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary());
} catch (NoSuchElementException e) {
entity.removeFromGraph();
}
}

100
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -26,12 +26,13 @@ import com.iqser.red.service.redaction.v1.model.ArgumentType;
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import lombok.Builder;
import lombok.Data;
@ -207,6 +208,7 @@ public class Section {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue()));
}
@SuppressWarnings("unused")
@WhenCondition
public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set<String> value) {
@ -214,6 +216,7 @@ public class Section {
return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue()));
}
@SuppressWarnings("unused")
@WhenCondition
public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) {
@ -571,12 +574,12 @@ public class Section {
@ThenAction
@SuppressWarnings("unused")
public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern,
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
@Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive,
@Argument(ArgumentType.INTEGER) int group,
@Argument(ArgumentType.TYPE) String asType,
@Argument(ArgumentType.RULE_NUMBER) int ruleNumber,
@Argument(ArgumentType.STRING) String reason,
@Argument(ArgumentType.LEGAL_BASIS) String legalBasis) {
redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false);
}
@ -665,7 +668,8 @@ public class Section {
legalBasis,
true,
skipRemoveEntitiesContainedInLarger,
sortedResult, false);
sortedResult,
false);
}
@ -697,11 +701,11 @@ public class Section {
legalBasis,
true,
skipRemoveEntitiesContainedInLarger,
sortedResult, ignoreTables);
sortedResult,
ignoreTables);
}
@ThenAction
@SuppressWarnings("unused")
public void redactBetween(@Argument(ArgumentType.STRING) String start,
@ -729,7 +733,8 @@ public class Section {
legalBasis,
true,
skipRemoveEntitiesContainedInLarger,
sortedResult, false);
sortedResult,
false);
}
@ -772,7 +777,8 @@ public class Section {
legalBasis,
true,
skipRemoveEntitiesContainedInLarger,
sortedResult, false);
sortedResult,
false);
}
}
@ -787,19 +793,7 @@ public class Section {
@Argument(ArgumentType.BOOLEAN) boolean redactEverywhere,
@Argument(ArgumentType.STRING) String reason) {
redactBetween(start,
stop,
false,
false,
asType,
ruleNumber,
redactEverywhere,
false,
reason,
null,
false,
false,
false, false);
redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, null, false, false, false, false);
}
@ -813,19 +807,7 @@ public class Section {
@Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine,
@Argument(ArgumentType.STRING) String reason) {
redactBetween(start,
stop,
false,
false,
asType,
ruleNumber,
redactEverywhere,
excludeHeadLine,
reason,
null,
false,
false,
false, false);
redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false, false, false);
}
@ -1293,14 +1275,14 @@ public class Section {
public Set<Entity> findEntities(String value,
String asType,
boolean caseInsensitive,
boolean redacted,
int ruleNumber,
String reason,
String legalBasis,
Engine engine,
boolean asRecommendation) {
String asType,
boolean caseInsensitive,
boolean redacted,
int ruleNumber,
String reason,
String legalBasis,
Engine engine,
boolean asRecommendation) {
String text = caseInsensitive ? searchText.toLowerCase() : searchText;
Set<Entity> found = EntitySearchUtils.findEntities(text,
@ -1445,7 +1427,8 @@ public class Section {
if (StringUtils.isNotBlank(stringOffset.getValue())) {
var trimmedOffsetString = stringOffset.trim();
Set<Entity> found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream()
.filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet());
.filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd())
.collect(Collectors.toSet());
found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger));
EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary);
@ -1459,7 +1442,15 @@ public class Section {
}
private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
private void redactByRegExWithNewlines(String pattern,
boolean patternCaseInsensitive,
int group,
String asType,
int ruleNumber,
String reason,
String legalBasis,
boolean redaction,
boolean skipRemoveEntitiesContainedInLarger) {
Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive);
@ -1476,7 +1467,15 @@ public class Section {
}
private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) {
private void redactByRegEx(String pattern,
boolean patternCaseInsensitive,
int group,
String asType,
int ruleNumber,
String reason,
String legalBasis,
boolean redaction,
boolean skipRemoveEntitiesContainedInLarger) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
@ -1524,8 +1523,7 @@ public class Section {
boolean sortedResult,
boolean ignoreTables) {
if(isInTable && ignoreTables){
if (isInTable && ignoreTables) {
return;
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java
View File

@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.model;
import java.util.HashMap;
import java.util.Map;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation;
import lombok.Data;
import lombok.NoArgsConstructor;
@ -12,4 +14,5 @@ public class TenantDictionary {
private final Map<String, DictionaryRepresentation> dictionariesByDossierTemplate = new HashMap<>();
private final Map<String, DictionaryRepresentation> dictionariesByDossier = new HashMap<>();
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import static java.lang.String.format;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.HashSet;
import java.util.Set;

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java
View File

@ -1,10 +1,10 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.Set;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.Set;
@Data
@AllArgsConstructor
public class DictionaryIncrement {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import lombok.AllArgsConstructor;
import lombok.Data;

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java
View File

@ -1,16 +1,15 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import lombok.AllArgsConstructor;
import lombok.Data;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;
import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import lombok.AllArgsConstructor;
import lombok.Data;
@Data
@AllArgsConstructor
public class DictionaryModel implements Serializable {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java
View File

@ -1,12 +1,12 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
import lombok.Data;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import lombok.Data;
@Data
public class DictionaryRepresentation {

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.model;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import lombok.AllArgsConstructor;
import lombok.Builder;

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.redaction.utils;
package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary;
import java.util.ArrayList;
import java.util.Collection;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java
View File

@ -22,14 +22,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import feign.FeignException;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -33,9 +33,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
@ -104,7 +104,7 @@ public class DroolsExecutionService {
kieSession.setGlobal("dictionary", dictionary);
document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
document.getPages().forEach(kieSession::insert);
fileAttributes.forEach(kieSession::insert);

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -9,6 +9,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea;
@ -18,12 +19,11 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
import io.micrometer.core.annotation.Timed;
import lombok.RequiredArgsConstructor;
@ -37,6 +37,7 @@ public class ManualRedactionSurroundingTextService {
private final RedactionStorageService redactionStorageService;
private final RedactionServiceSettings redactionServiceSettings;
@Timed("redactmanager_surroundingTextAnalysis")
public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) {
@ -87,9 +88,18 @@ public class ManualRedactionSurroundingTextService {
}
if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) {
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
SurroundingWordsService.addSurroundingText(Set.of(correctEntity),
sectionText.getSearchableText(),
null,
sectionText.getCellStarts(),
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
} else {
SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords());
SurroundingWordsService.addSurroundingText(Set.of(correctEntity),
sectionText.getSearchableText(),
null,
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords());
}
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());

10
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java
View File

@ -3,9 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service;
import java.util.List;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import io.micrometer.core.annotation.Timed;
import lombok.experimental.UtilityClass;
@ -15,7 +15,6 @@ import lombok.extern.slf4j.Slf4j;
@UtilityClass
public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingText")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
@ -38,7 +37,12 @@ public class SurroundingWordsService {
@Timed("redactmanager_addSurroundingTextTables")
public void addSurroundingText(Set<Entity> entities, SearchableText searchableText, Dictionary dictionary, List<Integer> cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) {
public void addSurroundingText(Set<Entity> entities,
SearchableText searchableText,
Dictionary dictionary,
List<Integer> cellstarts,
int surroundingWordsOffsetWindow,
int numberOfSurroundingWords) {
if (entities.isEmpty()) {
return;

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java
View File

@ -39,10 +39,10 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.exception.RedactionException;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService;
@ -86,8 +86,6 @@ public class AnalyzeService {
ImportedRedactionService importedRedactionService;
SectionFinder sectionFinder;
DocumentGraphFactory documentGraphFactory;
DocumentDataMapper documentDataMapper;
DocumentGraphMapper documentGraphMapper;
FunctionTimerValues redactmanagerAnalyzePagewiseValues;
@ -133,7 +131,7 @@ public class AnalyzeService {
sectionText.getSectionAreas())));
log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text));
redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid());
@ -154,7 +152,7 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
// not yet ready for reanalysis
if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) {
@ -208,7 +206,7 @@ public class AnalyzeService {
public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) {
long startTime = System.currentTimeMillis();
DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
NerEntities nerEntities;
if (redactionServiceSettings.isNerServiceEnabled()) {

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java
View File

@ -21,11 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import io.micrometer.core.annotation.Timed;
import lombok.AccessLevel;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java
View File

@ -11,12 +11,12 @@ import org.springframework.stereotype.Component;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entities;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails;
import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -13,9 +13,9 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService;
import lombok.AccessLevel;

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java
View File

@ -13,13 +13,14 @@ import java.util.stream.Collectors;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.Image;
import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java
View File

@ -34,7 +34,7 @@ public class PdfDraw {
public static void drawDocumentGraph(PDDocument document, DocumentGraph documentGraph) {
documentGraph.getTableOfContents().streamEntriesInOrder().forEach(entry -> drawNode(document, entry));
documentGraph.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> drawNode(document, entry));
}
@ -132,7 +132,8 @@ public class PdfDraw {
private static Options buildStandardOptionsForNodes(TableOfContents.Entry entry) {
return Options.builder().stroke(true).strokeColor(switch (entry.type()) {
return Options.builder().stroke(true).strokeColor(switch (entry.getType()) {
case DOCUMENT -> Color.LIGHT_GRAY;
case HEADER, FOOTER -> Color.GREEN;
case PARAGRAPH -> Color.BLUE;
case HEADLINE -> Color.RED;
@ -146,9 +147,9 @@ public class PdfDraw {
private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, TableOfContents.Entry entry, Options options) {
Map<PageNode, Rectangle2D> rectanglesPerPage = entry.node().getBBox();
Map<PageNode, Rectangle2D> rectanglesPerPage = entry.getNode().getBBox();
rectanglesPerPage.forEach((page, rectangle2D) -> {
if (entry.type() == NodeType.SECTION) {
if (entry.getType() == NodeType.SECTION) {
rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10);
}
drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options);
@ -159,7 +160,7 @@ public class PdfDraw {
private static String buildString(TableOfContents.Entry entry) {
return entry.node().getNumberOnPage() + ": " + entry.tocId() + ": " + entry.type().toString();
return entry.getNode().getNumberOnPage() + ": " + entry.getTocId() + ": " + entry.getType().toString();
}
}

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java
View File

@ -65,10 +65,10 @@ import lombok.SneakyThrows;
@Import(AbstractTestWithDictionaries.TestConfiguration.class)
public class AbstractTestWithDictionaries {
protected static final String RULES = loadFromClassPath("drools/rules.drl");
protected static final String RULES = loadFromClassPath("drools/testRules.drl");
protected static final String RULES_PATH = "drools/rules.drl";
protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl";
protected static final String MERGE_ENTITY_RULES_PATH = "drools/merge_entity_rules.drl";
protected static final String MERGE_ENTITY_RULES_PATH = "drools/testRules.drl";
private static final String VERTEBRATE = "vertebrate";
private static final String ADDRESS = "CBI_address";
private static final String AUTHOR = "CBI_author";
@ -518,7 +518,7 @@ public class AbstractTestWithDictionaries {
KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8"));
kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8"));
//kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8"));
KieRepository kieRepository = kieServices.getRepository();

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java
View File

@ -30,7 +30,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
import com.iqser.red.storage.commons.service.StorageService;

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -28,12 +28,12 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
@ -98,7 +98,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
long serializationStart = System.currentTimeMillis();
document.getEntities().forEach(kieSession::insert);
document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node()));
document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode()));
document.getPages().forEach(kieSession::insert);
System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart);
@ -140,7 +140,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).node();
SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).getNode();
start = System.currentTimeMillis();
for (int i = 0; i < n; i++) {
paragraph.buildTextBlock();

7
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java
View File

@ -1,5 +1,7 @@
package com.iqser.red.service.redaction.v1.server;
import static java.io.File.createTempFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -15,7 +17,6 @@ import org.springframework.core.io.InputStreamResource;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.commons.jackson.ObjectMapperFactory;
import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils;
import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist;
import com.iqser.red.storage.commons.service.StorageService;
@ -62,7 +63,7 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows
public <T> void storeJSONObject(String tenantId, String objectId, T any) {
File tempFile = FileUtils.createTempFile("test", ".tmp");
File tempFile = createTempFile("test", ".tmp");
getMapper().writeValue(new FileOutputStream(tempFile), any);
dataMap.put(objectId, tempFile);
}
@ -101,7 +102,7 @@ public class FileSystemBackedStorageService implements StorageService {
@SneakyThrows
public void storeObject(String tenantId, String objectId, InputStream stream) {
File tempFile = FileUtils.createTempFile("test", ".tmp");
File tempFile = createTempFile("test", ".tmp");
try (var fileOutputStream = new FileOutputStream(tempFile)) {
IOUtils.copy(stream, fileOutputStream);

1
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java
View File

@ -143,6 +143,7 @@ public class AnnotationService {
annotation.setTitlePopup(redactionLogEntry.getId());
annotation.setAnnotationName(redactionLogEntry.getId());
annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE));
annotation.setNoRotate(false);
annotations.add(annotation);
if (redactionLogEntry.getComments() != null) {

61
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java Normal file
View File

@ -0,0 +1,61 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows;
public class BuildDocumentGraphTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired
private ImageService imageService;
@Test
public void buildMetolachlor() {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
assertEquals(221, documentGraph.getPages().size());
assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count());
assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count());
}
@SneakyThrows
protected DocumentGraph buildGraph(String filename) {
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
} else {
prepareStorage(filename + ".pdf");
}
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<PdfImage>> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
return documentGraphFactory.buildDocumentGraph(classifiedDoc);
}
}
}

143
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionTest.java
View File

@ -4,18 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.wildfly.common.Assert.assertTrue;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode;
@ -26,25 +20,37 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
import com.iqser.red.service.redaction.v1.server.segmentation.ImageService;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows;
public class DocumentGraphTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest {
@Autowired
private EntityCreationService entityCreationService;
@Autowired
private ImageService imageService;
@Test
public void assertCollectAllEntitiesWorks() {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
createAndInsertEntity(documentGraph, "Clarissa");
createAndInsertEntity(documentGraph, "Lastname");
createAndInsertEntity(documentGraph, "David Ksenia");
createAndInsertEntity(documentGraph, "Michael N.");
createAndInsertEntity(documentGraph, "Page-Footer");
createAndInsertEntity(documentGraph, "CTL/with dictionary entry 1234 with Slash");
assertEquals(6, documentGraph.getEntities().size());
}
private EntityNode createAndInsertEntity(DocumentGraph documentGraph, String searchTerm) {
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
return entityNode;
}
@Test
@ -52,11 +58,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "Clarissa";
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assertEquals("Expand to Hint ", entityNode.getTextBefore());
assertEquals("’s Donut ←", entityNode.getTextAfter());
@ -67,7 +69,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@ -76,11 +78,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "Rule 39:";
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assertEquals("", entityNode.getTextBefore());
assertEquals(" Purity Hint", entityNode.getTextAfter());
@ -90,7 +88,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@ -99,11 +97,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
String searchTerm = "1998";
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assertEquals("", entityNode.getTextBefore());
assertEquals("", entityNode.getTextAfter());
@ -113,7 +107,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(15, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@ -134,9 +128,9 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/new/crafted document");
TableNode table = (TableNode) documentGraph.getTableOfContents()//
.streamEntriesInOrder()//
.filter(entry -> entry.type().equals(NodeType.TABLE))//
.map(TableOfContents.Entry::node)//
.streamAllEntriesInOrder()//
.filter(entry -> entry.getType().equals(NodeType.TABLE))//
.map(TableOfContents.Entry::getNode)//
.findFirst().orElseThrow();
assertEquals(5, table.getNumberOfCols());
assertEquals(4, table.getNumberOfRows());
@ -164,10 +158,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
TableNode table = (TableNode) documentGraph.getTableOfContents()
.streamEntriesInOrder()
.filter(entry -> entry.node().getPages().stream().anyMatch(page -> page.getNumber() == 22))
.filter(entry -> entry.type().equals(NodeType.TABLE))
.map(TableOfContents.Entry::node)
.streamAllEntriesInOrder()
.filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22))
.filter(entry -> entry.getType().equals(NodeType.TABLE))
.map(TableOfContents.Entry::getNode)
.findFirst()
.orElseThrow();
assertEquals(5, table.getNumberOfCols());
@ -185,11 +179,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
String searchTerm = "Cucurbit";
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
assertEquals(", Group 9;", entityNode.getTextAfter());
@ -197,9 +187,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(searchTerm, entityNode.getValue());
assertEquals(2, entityNode.getIntersectingNodes().size());
assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10));
assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@ -214,7 +205,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
assertEquals("2.6.1 Summary of ", entityNode.getTextBefore());
assertEquals(" and excretion in", entityNode.getTextAfter());
@ -222,10 +214,11 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
assertEquals(searchTerm, entityNode.getValue());
assertEquals(2, entityNode.getIntersectingNodes().size());
assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertEquals(4, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33));
assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@ -234,11 +227,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06");
String searchTerm = "N-deacetylation product";
int start = documentGraph.getTextBlock().indexOf(searchTerm);
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm);
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
assertEquals(" of metabolite of", entityNode.getTextAfter());
@ -246,43 +235,23 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assertEquals(3, entityNode.getIntersectingNodes().size());
assertEquals("2.7.2 Summary of metabolism, distribution and expression of residues in plants, poultry, lactating ruminants, pigs and fish ",
entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText());
// TODO: the number below is odd, due the cell (1, 2) containing 4 AtomicTextBlocks, even though it is an image. This should be fixed during structure analysis.
// In general, numberOnPage needs some more thought.
assertEquals(25, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54));
assertEquals(26, entityNode.getDeepestFullyContainingNode().getNumberOnPage());
assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode());
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
}
@SneakyThrows
protected DocumentGraph buildGraph(String filename) {
if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) {
prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json");
} else {
prepareStorage(filename + ".pdf");
}
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
try (InputStream inputStream = fileResource.getInputStream()) {
Map<Integer, List<PdfImage>> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID);
Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages);
return documentGraphFactory.buildDocumentGraph(classifiedDoc);
}
}
private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, int start, EntityNode entityNode) {
private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, EntityNode entityNode) {
List<Integer> paragraphStart = entityNode.getIntersectingNodes().stream()//
.map(SemanticNode::buildTextBlock)//
.map(textBlock -> textBlock.indexOf(searchTerm))//
.toList();
paragraphStart.forEach(nodeStart -> assertEquals(start, nodeStart));
paragraphStart.forEach(nodeStart -> assertEquals(entityNode.getBoundary().start(), nodeStart));
}
@ -293,15 +262,17 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY);
entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents());
PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
assertEquals(entityNode.getValue(), searchTerm);
assertTrue(pageNode.getEntities().contains(entityNode));
assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode)));
assertTrue(entityNode.getPages().contains(pageNode));
assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode);
assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode);
assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode)));
}
}

35
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java
View File

@ -1,37 +1,19 @@
package com.iqser.red.service.redaction.v1.server.document.graph;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData;
import com.iqser.red.service.redaction.v1.server.document.data.DocumentData;
import com.iqser.red.service.redaction.v1.server.document.data.PageData;
import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData;
import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper;
import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper;
import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext;
import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService;
import lombok.SneakyThrows;
public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
@Autowired
private DocumentGraphFactory documentGraphFactory;
@Autowired
private PdfSegmentationService segmentationService;
@Autowired
private DocumentDataMapper documentDataMapper;
@Autowired
private DocumentGraphMapper documentGraphMapper;
public class DocumentGraphMappingTest extends BuildDocumentGraphTest {
@Test
@SneakyThrows
@ -39,12 +21,9 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
String filename = "files/new/crafted document";
prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
DocumentGraph document = buildGraph(filename);
DocumentData documentData = DocumentDataMapper.toDocumentData(document);
var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null);
DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc);
DocumentData documentData = documentDataMapper.toDocumentData(document);
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks());
storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks());
@ -57,7 +36,13 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries {
AtomicPositionBlockData[].class);
TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class);
DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData);
DocumentData documentData2 = DocumentData.builder()
.pages(pageData)
.tableOfContents(tableOfContentsData)
.atomicTextBlocks(atomicTextBlockData)
.atomicPositionBlocks(atomicPositionBlockData)
.build();
DocumentGraph newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2);
assert document.toString().equals(newDocumentGraph.toString());
assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString());

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java
View File

@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw;
import lombok.SneakyThrows;
public class DocumentGraphVisualizationTest extends DocumentGraphTest {
public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest {
@Test
@SneakyThrows

45
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -18,18 +18,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "merge intersecting Entities of same type"
salience 100
@ -50,10 +53,10 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
when
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
entity.removeFromGraph();
retract(entity);
$entity.removeFromGraph();
retract($entity);
end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
@ -89,6 +92,8 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
retract($lowerRank);
end
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
@ -108,11 +113,11 @@ rule "1: Redact CBI Authors (Non vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
setFields($entity, 1, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
@ -120,11 +125,11 @@ rule "2: Redact CBI Authors (Vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
setFields($entity, 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
update($entity)
end
@ -132,7 +137,7 @@ rule "3: Don't redact CBI Address (Non vertebrate study)"
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
then
setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
@ -156,9 +161,9 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
insert(entity);
EntityNode falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields(falsePositive, 5, "Genitive Author", null, Engine.RULE);
insert(falsePositive);
end
@ -189,7 +194,7 @@ rule "7: Add CBI_author with \"et al.\" Regex"
rule "8: Add recommendation for Addresses in Test Organism sections"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
//FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
@ -228,7 +233,7 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study
no-loop true
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
then
$entity.setRedaction(true);
@ -253,7 +258,7 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
rule "12: Redact Emails by RegEx (Non vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
@ -277,7 +282,7 @@ rule "14: Redact line after contact information (Non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$string: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
@ -289,7 +294,7 @@ rule "14: Redact line after contact information (Non vertebrate study)"
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
@ -324,7 +329,7 @@ rule "15: Redact line after contact information (Vertebrate study)"
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
@ -345,7 +350,7 @@ rule "16: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Set<EntityNode> entities = new HashSet<>();