From a2776f24cd10f900e35f717ec1175a3ff8eee427 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Mon, 17 Apr 2023 12:54:17 +0200 Subject: [PATCH] RED-6369: Rules Refactor * refactored documentGraph.tableOfContents to include document as root --- .../server/classification/model/Document.java | 2 +- .../document/data/TableOfContentsData.java | 39 ++--- .../server/document/graph/DocumentGraph.java | 13 +- .../document/graph/TableOfContents.java | 118 +++++++++------ .../graph/factory/DocumentGraphFactory.java | 52 ++++--- .../server/document/graph/nodes/NodeType.java | 1 + .../document/graph/nodes/SemanticNode.java | 77 +++++----- .../document/mapper/DocumentDataMapper.java | 36 ++--- .../document/mapper/DocumentGraphMapper.java | 73 +++++---- .../services/EntityCreationService.java | 29 ++-- .../v1/server/redaction/model/Section.java | 100 ++++++------ .../redaction/model/TenantDictionary.java | 3 + .../model/{ => dictionary}/Dictionary.java | 2 +- .../{ => dictionary}/DictionaryEntries.java | 2 +- .../{ => dictionary}/DictionaryIncrement.java | 6 +- .../DictionaryIncrementValue.java | 2 +- .../{ => dictionary}/DictionaryModel.java | 13 +- .../DictionaryRepresentation.java | 6 +- .../{ => dictionary}/DictionaryVersion.java | 2 +- .../dictionary}/SearchImplementation.java | 2 +- .../redaction/service/DictionaryService.java | 14 +- .../service/DroolsExecutionService.java | 4 +- ...ManualRedactionSurroundingTextService.java | 18 ++- .../service/SurroundingWordsService.java | 10 +- .../service/analyze/AnalyzeService.java | 14 +- .../service/analyze/SectionFinder.java | 6 +- .../service/entityredaction/EntityFinder.java | 4 +- .../EntityRedactionService.java | 4 +- .../redaction/utils/EntitySearchUtils.java | 5 +- .../server/visualization/service/PdfDraw.java | 11 +- .../server/AbstractTestWithDictionaries.java | 6 +- .../v1/server/DictionaryServiceTest.java | 2 +- .../server/DocumentGraphIntegrationTest.java | 10 +- .../FileSystemBackedStorageService.java | 7 +- .../v1/server/RedactionIntegrationTest.java | 2 +- .../v1/server/annotate/AnnotationService.java | 1 + .../graph/BuildDocumentGraphTest.java | 61 ++++++++ ... => DocumentGraphEntityInsertionTest.java} | 143 +++++++----------- .../graph/DocumentGraphMappingTest.java | 35 ++--- .../graph/DocumentGraphVisualizationTest.java | 2 +- .../src/test/resources/drools/rules.drl | 45 +++--- 41 files changed, 531 insertions(+), 451 deletions(-) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/Dictionary.java (99%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryEntries.java (97%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryIncrement.java (95%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryIncrementValue.java (94%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryModel.java (97%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryRepresentation.java (97%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/{ => dictionary}/DictionaryVersion.java (95%) rename redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/{utils => model/dictionary}/SearchImplementation.java (97%) create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java rename redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/{DocumentGraphTest.java => DocumentGraphEntityInsertionTest.java} (69%) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java index 27cd8b3d..486a0d1d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java @@ -4,7 +4,7 @@ import java.util.ArrayList; import java.util.List; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionGrid; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java index e725c2ef..c8c50071 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/data/TableOfContentsData.java @@ -1,20 +1,16 @@ package com.iqser.red.service.redaction.v1.server.document.data; -import static java.lang.String.format; - -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Stream; -import javax.management.openmbean.InvalidKeyException; - import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; @@ -25,17 +21,17 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class TableOfContentsData { - List entries; + EntryData root; public EntryData get(List tocId) { - if (tocId.size() < 1) { - throw new InvalidKeyException(format("Section Identifier: \"%s\" is not valid.", tocId)); + if (tocId.isEmpty()) { + return root; } - EntryData entry = entries.get(tocId.get(0)); + EntryData entry = root.subEntries.get(tocId.get(0)); for (int id : tocId.subList(1, tocId.size())) { - entry = entry.subEntries().get(id); + entry = entry.subEntries.get(id); } return entry; } @@ -43,13 +39,7 @@ public class TableOfContentsData { public Stream streamAllEntries() { - return entries.stream().flatMap(TableOfContentsData::flatten); - } - - - private static List getIds(String idsAsString) { - - return Arrays.stream(idsAsString.split("\\.")).map(Integer::valueOf).toList(); + return Stream.concat(Stream.of(root), root.subEntries.stream()).flatMap(TableOfContentsData::flatten); } @@ -61,12 +51,23 @@ public class TableOfContentsData { private static Stream flatten(EntryData entry) { - return Stream.concat(Stream.of(entry), entry.subEntries().stream().flatMap(TableOfContentsData::flatten)); + return Stream.concat(Stream.of(entry), entry.subEntries.stream().flatMap(TableOfContentsData::flatten)); } @Builder - public record EntryData(NodeType type, int[] tocId, Long[] atomicBlocks, Long[] pages, Map properties, List subEntries) { + @Getter + @AllArgsConstructor + @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) + public static class EntryData { + + NodeType type; + int[] tocId; + Long[] atomicBlocks; + Long[] pages; + Map properties; + List subEntries; + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java index 580255a8..1084b763 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraph.java @@ -10,6 +10,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; +import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; @@ -20,13 +21,13 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.experimental.FieldDefaults; -import lombok.extern.slf4j.Slf4j; -@Slf4j @Data @Builder @AllArgsConstructor +@NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) public class DocumentGraph implements SemanticNode { @@ -44,7 +45,7 @@ public class DocumentGraph implements SemanticNode { public List getMainSections() { - return tableOfContents.entries.stream().filter(entry -> entry.node() instanceof SectionNode).map(entry -> (SectionNode) entry.node()).collect(Collectors.toList()); + return streamChildren().filter(node -> node instanceof SectionNode).map(node -> (SectionNode) node).collect(Collectors.toList()); } @@ -56,7 +57,7 @@ public class DocumentGraph implements SemanticNode { public Set getEntities() { - return streamAllNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet()); + return streamAllSubNodes().map(SemanticNode::getEntities).flatMap(Set::stream).collect(Collectors.toUnmodifiableSet()); } @@ -76,14 +77,14 @@ public class DocumentGraph implements SemanticNode { private Stream streamAllNodes() { - return tableOfContents.streamEntriesInOrder().map(TableOfContents.Entry::node); + return tableOfContents.streamAllEntriesInOrder().map(TableOfContents.Entry::getNode); } @Override public String toString() { - return tableOfContents.toString(); + return NodeType.DOCUMENT + ": " + buildTextBlock().buildSummary(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java index 74068f66..6b0db599 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/TableOfContents.java @@ -13,30 +13,33 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; +import lombok.experimental.FieldDefaults; @Data public class TableOfContents { - List entries; + private final Entry root; - public TableOfContents() { + public TableOfContents(DocumentGraph documentGraph) { - entries = new LinkedList<>(); + root = Entry.builder().tocId(Collections.emptyList()).type(NodeType.DOCUMENT).children(new LinkedList<>()).node(documentGraph).build(); } public TextBlock buildTextBlock() { - return streamEntriesInOrder().map(Entry::node).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector()); + return streamAllEntriesInOrder().map(Entry::getNode).filter(SemanticNode::isTerminal).map(SemanticNode::getTerminalTextBlock).collect(new TextBlockCollector()); } - public List createNewEntryAndReturnId(NodeType nodeType, SemanticNode node) { + public List createNewMainEntryAndReturnId(NodeType nodeType, SemanticNode node) { return createNewChildEntryAndReturnId(Collections.emptyList(), nodeType, node); } @@ -44,32 +47,30 @@ public class TableOfContents { public List createNewChildEntryAndReturnId(List parentId, NodeType nodeType, SemanticNode node) { - List newId; - if (entryExists(parentId)) { - Entry parent = getEntryById(parentId); - newId = new LinkedList<>(parentId); - newId.add(parent.children().size()); - parent.children().add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build()); - } else { - newId = List.of(entries.size()); - entries.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build()); + if (!entryExists(parentId)) { + throw new UnsupportedOperationException(format("parentId %s does not exist!", parentId)); } + Entry parent = getEntryById(parentId); + List newId = new LinkedList<>(parentId); + newId.add(parent.children.size()); + parent.children.add(Entry.builder().tocId(newId).node(node).type(nodeType).children(new LinkedList<>()).build()); + return newId; } private boolean entryExists(List tocId) { - if (tocId.size() < 1) { - return false; + if (tocId.isEmpty()) { + return root != null; } - Entry entry = entries.get(tocId.get(0)); + Entry entry = root.children.get(tocId.get(0)); for (int id : tocId.subList(1, tocId.size())) { if (id >= entry.children.size() || 0 > id) { return false; } - entry = entry.children().get(id); + entry = entry.children.get(id); } return true; } @@ -77,81 +78,103 @@ public class TableOfContents { public Entry getParentEntryById(List tocId) { - List parentIds = getParentId(tocId); - if (parentIds.size() < 1) { - throw new NotFoundException(format("Node with tocId \"%s\" has no parent!", tocId)); - } - return getEntryById(parentIds); + return getEntryById(getParentId(tocId)); } public boolean hasParentById(List tocId) { - List parentId = getParentId(tocId); - return entryExists(parentId); + return entryExists(getParentId(tocId)); } - public Stream streamChildren(List tocId) { + public Stream streamChildrenNodes(List tocId) { - return getEntryById(tocId).children().stream().map(Entry::node); + return getEntryById(tocId).children.stream().map(Entry::getNode); } private static List getParentId(List tocId) { + if (tocId.isEmpty()) { + throw new UnsupportedOperationException("Root has no parent!"); + } + if (tocId.size() < 2) { + return Collections.emptyList(); + } return tocId.subList(0, tocId.size() - 1); } public Entry getEntryById(List tocId) { - Entry entry = entries.get(tocId.get(0)); + if (tocId.isEmpty()) { + return root; + } + Entry entry = root.children.get(tocId.get(0)); for (int id : tocId.subList(1, tocId.size())) { - entry = entry.children().get(id); + entry = entry.children.get(id); } return entry; } - public Stream streamEntriesInOrder() { + public Stream streamMainEntries() { - return entries.stream().flatMap(TableOfContents::flatten); + return root.children.stream(); } - public Stream streamSubEntriesInOrder(List parentId) { + public Stream streamAllEntriesInOrder() { - return Stream.of(getEntryById(parentId)).flatMap(TableOfContents::flatten); + return Stream.of(root).flatMap(TableOfContents::flatten); + } + + + public Stream streamAllSubEntriesInOrder(List parentId) { + + return getEntryById(parentId).children.stream().flatMap(TableOfContents::flatten); } @Override public String toString() { - return String.join("\n", streamEntriesInOrder().map(Entry::toString).toList()); - } - - - public String toString(List id) { - - return String.join("\n", streamSubEntriesInOrder(id).map(Entry::toString).toList()); + return String.join("\n", streamAllEntriesInOrder().map(Entry::toString).toList()); } private static Stream flatten(Entry entry) { - return Stream.concat(Stream.of(entry), entry.children().stream().flatMap(TableOfContents::flatten)); + return Stream.concat(Stream.of(entry), entry.children.stream().flatMap(TableOfContents::flatten)); + } + + + public SemanticNode getHighestParentById(List tocId) { + + if (tocId.isEmpty()) { + return root.node; + } + return root.children.get(tocId.get(0)).node; } @Builder - public record Entry(List tocId, NodeType type, SemanticNode node, List children) { + @Getter + @AllArgsConstructor + @FieldDefaults(level = AccessLevel.PRIVATE, makeFinal = true) + public static class Entry { + + List tocId; + NodeType type; + SemanticNode node; + List children; + @Override public String toString() { - return node().toString(); + return node.toString(); } @@ -161,6 +184,13 @@ public class TableOfContents { return Hashing.murmur3_32_fixed().hashString(toString(), StandardCharsets.UTF_8).hashCode(); } + + @Override + public boolean equals(Object o) { + + return o instanceof Entry && o.hashCode() == this.hashCode(); + } + } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java index 2e72ff68..e52bf261 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/factory/DocumentGraphFactory.java @@ -1,10 +1,5 @@ package com.iqser.red.service.redaction.v1.server.document.graph.factory; -import static com.iqser.red.service.redaction.v1.server.document.graph.factory.RectangleTransformations.toRectangle2D; -import static com.iqser.red.service.redaction.v1.server.document.graph.factory.TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX; -import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER; -import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER; -import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.TABLE_CELL; import static java.lang.String.format; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; @@ -15,10 +10,10 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import com.iqser.red.service.redaction.v1.server.classification.model.Document; @@ -40,7 +35,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNo import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableCellNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer; @@ -56,15 +50,17 @@ public class DocumentGraphFactory { public DocumentGraph buildDocumentGraph(Document document) { TextBlockFactory textBlockFactory = new TextBlockFactory(); - Context context = new Context(new TableOfContents(), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory); + DocumentGraph documentGraph = new DocumentGraph(); + Context context = new Context(new TableOfContents(documentGraph), new HashMap<>(), new LinkedList<>(), new LinkedList<>(), textBlockFactory); document.getPages().stream().map(this::buildPage).forEach(page -> context.pages().put(page, new AtomicInteger(1))); document.getSections().stream().flatMap(section -> section.getImages().stream()).forEach(image -> context.images().add(image)); addSections(document, context); addHeaderAndFooterToEachPage(document, context); - DocumentGraph documentGraph = DocumentGraph.builder().numberOfPages(context.pages.size()).pages(context.pages.keySet()).tableOfContents(context.tableOfContents).build(); - + documentGraph.setNumberOfPages(context.pages.size()); + documentGraph.setPages(context.pages.keySet()); + documentGraph.setTableOfContents(context.tableOfContents); documentGraph.setTextBlock(documentGraph.buildTextBlock()); return documentGraph; } @@ -86,7 +82,7 @@ public class DocumentGraphFactory { List tocId; if (parentNode == null) { - tocId = context.tableOfContents.createNewEntryAndReturnId(NodeType.SECTION, sectionNode); + tocId = context.tableOfContents.createNewMainEntryAndReturnId(NodeType.SECTION, sectionNode); } else { tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.SECTION, sectionNode); } @@ -177,7 +173,7 @@ public class DocumentGraphFactory { com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock textBlock; - List tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), TABLE_CELL, tableCellNode); + List tocId = context.tableOfContents().createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.TABLE_CELL, tableCellNode); tableCellNode.setTocId(tocId); if (cell.getTextBlocks().isEmpty()) { @@ -194,7 +190,7 @@ public class DocumentGraphFactory { tableCellNode.setTerminal(false); } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { - List sequences = mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); + List sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); textBlock = context.textBlockFactory().buildAtomicTextBlock(sequences, tableCellNode, context, page); tableCellNode.setTerminalTextBlock(textBlock); tableCellNode.setTerminal(true); @@ -215,7 +211,8 @@ public class DocumentGraphFactory { private static boolean firstTextBlockIsHeadline(Cell cell) { - return StringUtils.startsWith(cell.getTextBlocks().get(0).getClassification(), "H"); + String classification = cell.getTextBlocks().get(0).getClassification(); + return classification != null && classification.startsWith("H"); } @@ -230,7 +227,7 @@ public class DocumentGraphFactory { PageNode page = getPage(originalTextBlock.getPage(), context); SemanticNode node; - if (StringUtils.startsWith(originalTextBlock.getClassification(), "H")) { + if (originalTextBlock.getClassification() != null && originalTextBlock.getClassification().startsWith("H")) { node = HeadlineNode.builder().tableOfContents(context.tableOfContents()).build(); } else { node = ParagraphNode.builder().tableOfContents(context.tableOfContents()).build(); @@ -240,7 +237,7 @@ public class DocumentGraphFactory { List textBlocks = new LinkedList<>(textBlocksToMerge); textBlocks.add(originalTextBlock); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); + AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); if (node instanceof HeadlineNode headlineNode) { List tocId = context.tableOfContents.createNewChildEntryAndReturnId(parentNode.getTocId(), NodeType.HEADLINE, node); @@ -260,7 +257,7 @@ public class DocumentGraphFactory { PageNode page = getPage(image.getPage(), context); ImageNode imageNode = ImageNode.builder() .imageType(image.getImageType()) - .position(toRectangle2D(image.getPosition())) + .position(RectangleTransformations.toRectangle2D(image.getPosition())) .transparency(image.isHasTransparency()) .page(page) .tableOfContents(context.tableOfContents()) @@ -308,8 +305,11 @@ public class DocumentGraphFactory { PageNode page = getPage(textBlocks.get(0).getPage(), context); FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page); - List tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); + AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), + footer, + context, + page); + List tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer); footer.setTocId(tocId); footer.setTerminalTextBlock(textBlock); page.setFooter(footer); @@ -320,8 +320,12 @@ public class DocumentGraphFactory { PageNode page = getPage(textBlocks.get(0).getPage(), context); HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, context, 0, page); - List tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); + AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), + header, + context, + 0, + page); + List tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header); header.setTocId(tocId); header.setTerminalTextBlock(textBlock); page.setHeader(header); @@ -333,7 +337,7 @@ public class DocumentGraphFactory { PageNode page = getPage(pageIndex, context); FooterNode footer = FooterNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(footer, context, page); - List tocId = context.tableOfContents().createNewEntryAndReturnId(FOOTER, footer); + List tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.FOOTER, footer); footer.setTocId(tocId); footer.setTerminalTextBlock(textBlock); page.setFooter(footer); @@ -345,7 +349,7 @@ public class DocumentGraphFactory { PageNode page = getPage(pageIndex, context); HeaderNode header = HeaderNode.builder().tableOfContents(context.tableOfContents()).build(); AtomicTextBlock textBlock = context.textBlockFactory.emptyTextBlock(header, 0, page); - List tocId = context.tableOfContents().createNewEntryAndReturnId(HEADER, header); + List tocId = context.tableOfContents().createNewMainEntryAndReturnId(NodeType.HEADER, header); header.setTocId(tocId); header.setTerminalTextBlock(textBlock); page.setHeader(header); @@ -370,7 +374,7 @@ public class DocumentGraphFactory { .stream() .filter(page -> page.getNumber() == pageIndex) .findFirst() - .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); + .orElseThrow(() -> new NoSuchElementException(format("ClassificationPage with number %d not found", pageIndex))); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java index 0da9d4e3..48b1f7bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/NodeType.java @@ -1,6 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; public enum NodeType { + DOCUMENT, SECTION, HEADLINE, PARAGRAPH, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java index f3dd6ce8..ba011ff3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/graph/nodes/SemanticNode.java @@ -1,12 +1,11 @@ package com.iqser.red.service.redaction.v1.server.document.graph.nodes; -import static java.util.stream.Collectors.groupingBy; - import java.awt.geom.Rectangle2D; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; @@ -20,10 +19,10 @@ public interface SemanticNode { /** * Searches all Nodes located underneath this Node in the TableOfContents and concatenates their AtomicTextBlocks into a single TextBlockEntity. - * So, for a Section all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity + * So, for a ClassificationSection all TextBlocks of Subsections, Paragraphs, and Tables are concatenated into a single TextBlockEntity * If the Node is Terminal, the TerminalTextBlock will be returned instead. * - * @return TextBlock containing all AtomicTextBlocks that are located under this Node. + * @return ClassificationTextBlock containing all AtomicTextBlocks that are located under this Node. */ TextBlock buildTextBlock(); @@ -38,7 +37,7 @@ public interface SemanticNode { /** - * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's TextBlock + * Each AtomicTextBlock is assigned a page, so to get the pages this node appears on, it collects the PageNodes from each AtomicTextBlock belonging to this node's ClassificationTextBlock. * * @return Set of PageNodes this node appears on. */ @@ -49,13 +48,13 @@ public interface SemanticNode { /** - * @return the TableOfContents of the Document this node belongs to + * @return the TableOfContents of the ClassificationDocument this node belongs to */ TableOfContents getTableOfContents(); /** - * The id is a List of Integers uniquely identifying this node in the TableOfContents + * The id is a List of Integers uniquely identifying this node in the TableOfContents. * * @return the TableOfContents ID */ @@ -63,7 +62,7 @@ public interface SemanticNode { /** - * This should only be used during graph construction + * This should only be used during graph construction. * * @param tocId List of Integers */ @@ -97,16 +96,26 @@ public interface SemanticNode { */ default SemanticNode getParent() { - return getTableOfContents().getParentEntryById(getTocId()).node(); + return getTableOfContents().getParentEntryById(getTocId()).getNode(); } /** - * Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. + * @return The SemanticNode which is directly underneath the document and also under which this node is. + * if this is the highest child node or the document itself, it returns itself. + */ + default SemanticNode getHighestParent() { + + return getTableOfContents().getHighestParentById(getTocId()); + } + + + /** + * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden. * Currently only Sections, Images, and Tables are not terminal. * A TableCell might be Terminal depending on its area compared to the page. * - * @return boolean, indicating if a Node has direct access to a TextBlock + * @return boolean, indicating if a Node has direct access to a ClassificationTextBlock */ default boolean isTerminal() { @@ -115,7 +124,7 @@ public interface SemanticNode { /** - * Terminal means a SemanticNode has direct access to a TextBlock, by default this is false and must be overridden. + * Terminal means a SemanticNode has direct access to a ClassificationTextBlock, by default this is false and must be overridden. * Currently only Sections and Tables are not terminal. * * @return AtomicTextBlock @@ -150,7 +159,7 @@ public interface SemanticNode { /** - * @return true, if this node's TextBlock is not empty + * @return true, if this node's ClassificationTextBlock is not empty */ default boolean hasText() { @@ -159,8 +168,8 @@ public interface SemanticNode { /** - * @param string A String which the TextBlock might contain - * @return true, if this node's TextBlock contains the string + * @param string A String which the ClassificationTextBlock might contain + * @return true, if this node's ClassificationTextBlock contains the string */ default boolean containsString(String string) { @@ -169,8 +178,8 @@ public interface SemanticNode { /** - * @param strings A List of Strings which the TextBlock might contain - * @return true, if this node's TextBlock contains any of the strings + * @param strings A List of Strings which the ClassificationTextBlock might contain + * @return true, if this node's ClassificationTextBlock contains any of the strings */ default boolean containsAnyString(List strings) { @@ -179,34 +188,34 @@ public interface SemanticNode { /** - * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the Entity intersects or even contains the Entity. + * This function is used during insertion of EntityNodes into the graph, it checks if the boundary of the EntityNode intersects or even contains the EntityNode. * It sets the fields accordingly and recursively calls this function on all its children. * - * @param entity EntityNode, which is being inserted into the graph + * @param entityNode EntityNode, which is being inserted into the graph */ - default void addThisToEntityIfIntersects(EntityNode entity) { + default void addThisToEntityIfIntersects(EntityNode entityNode) { TextBlock textBlock = buildTextBlock(); - if (textBlock.getBoundary().intersects(entity.getBoundary())) { + if (textBlock.getBoundary().intersects(entityNode.getBoundary())) { - if (textBlock.containsBoundary(entity.getBoundary())) { - entity.setDeepestFullyContainingNode(this); + if (textBlock.containsBoundary(entityNode.getBoundary())) { + entityNode.setDeepestFullyContainingNode(this); } - entity.addIntersectingNode(this); - streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entity)); + entityNode.addIntersectingNode(this); + streamChildren().forEach(node -> node.addThisToEntityIfIntersects(entityNode)); } } /** - * Streams all children located directly underneath this node in the TableOfContents + * Streams all children located directly underneath this node in the TableOfContents. * * @return Stream of all children */ default Stream streamChildren() { - return getTableOfContents().streamChildren(getTocId()); + return getTableOfContents().streamChildrenNodes(getTocId()); } @@ -217,12 +226,12 @@ public interface SemanticNode { */ default Stream streamAllSubNodes() { - return getTableOfContents().streamSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::node); + return getTableOfContents().streamAllSubEntriesInOrder(getTocId()).map(TableOfContents.Entry::getNode); } /** - * @return Boundary of this Node's TextBlock + * @return Boundary of this Node's ClassificationTextBlock */ default Boundary getBoundary() { @@ -232,7 +241,7 @@ public interface SemanticNode { /** * If this Node is Terminal it will calculate the boundingBox of its TerminalTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children. - * If called on the Document, it will return the cropbox of each page + * If called on the ClassificationDocument, it will return the cropbox of each page * * @return Rectangle2D fully encapsulating this Node for each page. */ @@ -248,7 +257,7 @@ public interface SemanticNode { /** - * TODO this does not yet work for sections spanning multiple columns + * TODO this does not yet work for sections spanning multiple columns. * * @param bBoxPerPage initial empty BoundingBox * @return The union of the BoundingBoxes of all children @@ -264,13 +273,13 @@ public interface SemanticNode { /** * @param bBoxPerPage initial empty BoundingBox - * @return The union of all BoundingBoxes of the TextBlock of this node + * @return The union of all BoundingBoxes of the ClassificationTextBlock of this node */ private Map getBBoxFromTerminalTextBlock(Map bBoxPerPage) { - Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(groupingBy(AtomicTextBlock::getPage)); + Map> atomicTextBlockPerPage = buildTextBlock().getAtomicTextBlocks().stream().collect(Collectors.groupingBy(AtomicTextBlock::getPage)); atomicTextBlockPerPage.forEach((page, atbs) -> bBoxPerPage.put(page, RectangleTransformations.bBoxUnionAtomicTextBlock(atbs))); return bBoxPerPage; } -} +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java index e2a102c8..bc156202 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentDataMapper.java @@ -5,8 +5,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.springframework.stereotype.Service; - import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; @@ -21,7 +19,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -@Service +import lombok.experimental.UtilityClass; + +@UtilityClass public class DocumentDataMapper { public DocumentData toDocumentData(DocumentGraph documentGraph) { @@ -29,16 +29,16 @@ public class DocumentDataMapper { List atomicTextBlockData = documentGraph.streamTerminalTextBlocksInOrder() .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) .distinct() - .map(this::toAtomicTextBlockData) + .map(DocumentDataMapper::toAtomicTextBlockData) .toList(); List atomicPositionBlockData = documentGraph.streamTerminalTextBlocksInOrder() .flatMap(textBlock -> textBlock.getAtomicTextBlocks().stream()) .distinct() - .map(this::toAtomicPositionBlockData) + .map(DocumentDataMapper::toAtomicPositionBlockData) .toList(); - List pageData = documentGraph.getPages().stream().map(this::toPageData).toList(); + List pageData = documentGraph.getPages().stream().map(DocumentDataMapper::toPageData).toList(); TableOfContentsData tableOfContentsData = toTableOfContentsData(documentGraph.getTableOfContents()); return DocumentData.builder() .atomicTextBlocks(atomicTextBlockData.toArray(new AtomicTextBlockData[0])) @@ -51,7 +51,7 @@ public class DocumentDataMapper { private TableOfContentsData toTableOfContentsData(TableOfContents tableOfContents) { - return new TableOfContentsData(tableOfContents.getEntries().stream().map(this::toEntryData).toList()); + return new TableOfContentsData(toEntryData(tableOfContents.getRoot())); } @@ -59,25 +59,25 @@ public class DocumentDataMapper { Long[] atomicTextBlocks; - if (entry.node().isTerminal()) { - atomicTextBlocks = toAtomicTextBlockIds(entry.node().getTerminalTextBlock()); + if (entry.getNode().isTerminal()) { + atomicTextBlocks = toAtomicTextBlockIds(entry.getNode().getTerminalTextBlock()); } else { atomicTextBlocks = new Long[]{}; } - Map properties = switch (entry.type()) { - case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.node()); - case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.node()); - case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.node()); + Map properties = switch (entry.getType()) { + case TABLE -> PropertiesMapper.buildTableProperties((TableNode) entry.getNode()); + case TABLE_CELL -> PropertiesMapper.buildTableCellProperties((TableCellNode) entry.getNode()); + case IMAGE -> PropertiesMapper.buildImageProperties((ImageNode) entry.getNode()); default -> new HashMap<>(); }; return TableOfContentsData.EntryData.builder() - .tocId(toPrimitiveIntArray(entry.tocId())) - .subEntries(entry.children().stream().map(this::toEntryData).toList()) - .type(entry.type()) + .tocId(toPrimitiveIntArray(entry.getTocId())) + .subEntries(entry.getChildren().stream().map(DocumentDataMapper::toEntryData).toList()) + .type(entry.getType()) .atomicBlocks(atomicTextBlocks) - .pages(entry.node().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new)) + .pages(entry.getNode().getPages().stream().map(PageNode::getNumber).map(Integer::longValue).toArray(Long[]::new)) .properties(properties) .build(); } @@ -143,4 +143,4 @@ public class DocumentDataMapper { return array; } -} +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java index 8af164b1..86602d7c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/mapper/DocumentGraphMapper.java @@ -1,10 +1,7 @@ package com.iqser.red.service.redaction.v1.server.document.mapper; -import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseImageProperties; -import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableCellProperties; -import static com.iqser.red.service.redaction.v1.server.document.mapper.PropertiesMapper.parseTableProperties; -import static java.lang.Math.toIntExact; -import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.FOOTER; +import static com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType.HEADER; import java.awt.geom.Rectangle2D; import java.util.Arrays; @@ -12,9 +9,7 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; - -import org.apache.commons.lang3.NotImplementedException; -import org.springframework.stereotype.Service; +import java.util.NoSuchElementException; import com.google.common.primitives.Ints; import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; @@ -29,7 +24,6 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.FooterNode import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeaderNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ImageNode; -import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.ParagraphNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; @@ -39,29 +33,30 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlockCollector; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; -@Service +import lombok.experimental.UtilityClass; + +@UtilityClass public class DocumentGraphMapper { public DocumentGraph toDocumentGraph(DocumentData documentData) { + DocumentGraph documentGraph = new DocumentGraph(); Context context = new Context(documentData, - new TableOfContents(), + new TableOfContents(documentGraph), new LinkedList<>(), new LinkedList<>(), Arrays.stream(documentData.getAtomicTextBlocks()).toList(), Arrays.stream(documentData.getAtomicPositionBlocks()).toList()); - context.pages.addAll(Arrays.stream(documentData.getPages()).map(this::buildPage).toList()); + context.pages.addAll(Arrays.stream(documentData.getPages()).map(DocumentGraphMapper::buildPage).toList()); - context.tableOfContents.setEntries(buildEntries(documentData.getTableOfContents().getEntries(), context)); + context.tableOfContents.getRoot().getChildren().addAll(buildEntries(documentData.getTableOfContents().getRoot().getSubEntries(), context)); + + documentGraph.setTableOfContents(context.tableOfContents); + documentGraph.setPages(new HashSet<>(context.pages)); + documentGraph.setNumberOfPages(documentData.getPages().length); - DocumentGraph documentGraph = DocumentGraph.builder() - .numberOfPages(documentData.getPages().length) - .pages(new HashSet<>(context.pages)) - .tableOfContents(context.tableOfContents) - .build(); documentGraph.setTextBlock(documentGraph.buildTextBlock()); return documentGraph; } @@ -73,35 +68,35 @@ public class DocumentGraphMapper { for (TableOfContentsData.EntryData entryData : entries) { boolean terminal = isTerminal(entryData); - List pages = Arrays.stream(entryData.pages()).map(pageNumber -> getPage(pageNumber, context)).toList(); + List pages = Arrays.stream(entryData.getPages()).map(pageNumber -> getPage(pageNumber, context)).toList(); - SemanticNode node = switch (entryData.type()) { + SemanticNode node = switch (entryData.getType()) { case SECTION -> buildSection(context); case PARAGRAPH -> buildParagraph(context, terminal); case HEADLINE -> buildHeadline(context, terminal); case HEADER -> buildHeader(context, terminal); case FOOTER -> buildFooter(context, terminal); - case TABLE -> buildTable(context, entryData.properties()); - case TABLE_CELL -> buildTableCell(context, entryData.properties(), terminal); - case IMAGE -> buildImage(context, entryData.properties()); - default -> throw new NotImplementedException("Not yet implemented for type " + entryData.type()); + case TABLE -> buildTable(context, entryData.getProperties()); + case TABLE_CELL -> buildTableCell(context, entryData.getProperties(), terminal); + case IMAGE -> buildImage(context, entryData.getProperties()); + default -> throw new UnsupportedOperationException("Not yet implemented for type " + entryData.getType()); }; if (node.isTerminal()) { - TextBlock textBlock = toTextBlock(entryData.atomicBlocks(), context, node); + TextBlock textBlock = toTextBlock(entryData.getAtomicBlocks(), context, node); node.setTerminalTextBlock(textBlock); } - List tocId = Arrays.stream(entryData.tocId()).boxed().toList(); + List tocId = Arrays.stream(entryData.getTocId()).boxed().toList(); node.setTocId(tocId); - if (entryData.type() == NodeType.HEADER) { + if (entryData.getType() == HEADER) { pages.forEach(page -> page.setHeader((HeaderNode) node)); - } else if (entryData.type() == NodeType.FOOTER) { + } else if (entryData.getType() == FOOTER) { pages.forEach(page -> page.setFooter((FooterNode) node)); } else { pages.forEach(page -> page.getMainBody().add(node)); } - newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.type()).children(buildEntries(entryData.subEntries(), context)).node(node).build()); + newEntries.add(TableOfContents.Entry.builder().tocId(tocId).type(entryData.getType()).children(buildEntries(entryData.getSubEntries(), context)).node(node).build()); } return newEntries; } @@ -115,14 +110,14 @@ public class DocumentGraphMapper { private static boolean isTerminal(TableOfContentsData.EntryData entryData) { - return entryData.atomicBlocks().length > 0; + return entryData.getAtomicBlocks().length > 0; } private ImageNode buildImage(Context context, Map properties) { var builder = ImageNode.builder(); - parseImageProperties(properties, builder); + PropertiesMapper.parseImageProperties(properties, builder); return builder.tableOfContents(context.tableOfContents()).build(); } @@ -130,7 +125,7 @@ public class DocumentGraphMapper { private TableCellNode buildTableCell(Context context, Map properties, boolean terminal) { TableCellNode.TableCellNodeBuilder builder = TableCellNode.builder(); - parseTableCellProperties(properties, builder); + PropertiesMapper.parseTableCellProperties(properties, builder); return builder.terminal(terminal).tableOfContents(context.tableOfContents()).build(); } @@ -138,7 +133,7 @@ public class DocumentGraphMapper { private TableNode buildTable(Context context, Map properties) { TableNode.TableNodeBuilder builder = TableNode.builder(); - parseTableProperties(properties, builder); + PropertiesMapper.parseTableProperties(properties, builder); return TableNode.builder().tableOfContents(context.tableOfContents()).build(); } @@ -171,8 +166,8 @@ public class DocumentGraphMapper { private TextBlock toTextBlock(Long[] atomicTextBlockIds, Context context, SemanticNode parent) { return Arrays.stream(atomicTextBlockIds) - .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(toIntExact(atomicTextBlockId)), - context.atomicPositionBlockData.get(toIntExact(atomicTextBlockId)), + .map(atomicTextBlockId -> toAtomicTextBlock(context.atomicTextBlockData.get(Math.toIntExact(atomicTextBlockId)), + context.atomicPositionBlockData.get(Math.toIntExact(atomicTextBlockId)), parent, context)) .collect(new TextBlockCollector()); @@ -210,14 +205,14 @@ public class DocumentGraphMapper { private PageNode getPage(Long pageIndex, Context context) { return context.pages.stream() - .filter(page -> page.getNumber() == toIntExact(pageIndex)) + .filter(page -> page.getNumber() == Math.toIntExact(pageIndex)) .findFirst() - .orElseThrow(() -> new NotFoundException(format("Page with number %d not found", pageIndex))); + .orElseThrow(() -> new NoSuchElementException(String.format("ClassificationPage with number %d not found", pageIndex))); } record Context( - DocumentData documentData, + DocumentData layoutParsingModel, TableOfContents tableOfContents, List pages, List sections, diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index 31c89306..fcca9bce 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -5,8 +5,10 @@ import static com.iqser.red.service.redaction.v1.server.redaction.utils.Separato import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.isWhiteSpacesOrSeparatorsOnly; import java.util.Collections; +import java.util.Comparator; import java.util.LinkedList; import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -19,9 +21,8 @@ import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNod import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; -import com.iqser.red.service.redaction.v1.server.exception.NotFoundException; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -122,14 +123,18 @@ public class EntityCreationService { if (!allEntitiesIntersectAndHaveSameTypes(entitiesToMerge)) { throw new IllegalArgumentException("Provided entities can not be merged!"); } + if (entitiesToMerge.size() < 2) { + throw new IllegalArgumentException("more than 2 entities are required to merge!"); + } + EntityNode mergedEntity = EntityNode.initialEntityNode(Boundary.merge(entitiesToMerge.stream().map(EntityNode::getBoundary).toList()), type, entityType); mergedEntity.setRedaction(entitiesToMerge.stream().anyMatch(EntityNode::isRedaction)); mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet())); - if (mergedEntity.isRedaction()) { - mergedEntity.setRedactionReason(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getRedactionReason).findFirst().orElse("")); - mergedEntity.setMatchedRule(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getMatchedRule).findFirst().orElse(-1)); - mergedEntity.setLegalBasis(entitiesToMerge.stream().filter(EntityNode::isRedaction).map(EntityNode::getLegalBasis).findFirst().orElse("")); - } + EntityNode entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(EntityNode::getMatchedRule)).orElse(entitiesToMerge.get(0)); + mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason()); + mergedEntity.setMatchedRule(entityWithHigherRuleNumber.getMatchedRule()); + mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis()); + addEntityToGraph(mergedEntity, node.getTableOfContents()); return mergedEntity; } @@ -144,12 +149,10 @@ public class EntityCreationService { public void addEntityToGraph(EntityNode entity, TableOfContents tableOfContents) { try { - SemanticNode containingNode = tableOfContents.getEntries() - .stream() - .map(TableOfContents.Entry::node) + SemanticNode containingNode = tableOfContents.streamChildrenNodes(Collections.emptyList()) .filter(node -> node.buildTextBlock().containsBoundary(entity.getBoundary())) .findFirst() - .orElseThrow(() -> new NotFoundException("No containing Node found!")); + .orElseThrow(() -> new NoSuchElementException("No containing Node found!")); containingNode.addThisToEntityIfIntersects(entity); @@ -159,9 +162,7 @@ public class EntityCreationService { addToPages(entity); addToNodeEntitySets(entity); - } catch (NotFoundException e) { - entityEnrichmentService.enrichEntity(entity, tableOfContents.buildTextBlock()); - log.warn("Entity \"{}\" with {} is in between two main sections and will be removed!", entity.getValue(), entity.getBoundary()); + } catch (NoSuchElementException e) { entity.removeFromGraph(); } } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 27aa9cfc..49d76e7d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -26,12 +26,13 @@ import com.iqser.red.service.redaction.v1.model.ArgumentType; import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock; import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition; import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder; import com.iqser.red.service.redaction.v1.server.redaction.utils.OffsetStringUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import lombok.Builder; import lombok.Data; @@ -207,6 +208,7 @@ public class Section { return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.equals(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition public boolean fileAttributeContainsAnyOf(@Argument(ArgumentType.FILE_ATTRIBUTE) String label, @Argument(ArgumentType.STRING) Set value) { @@ -214,6 +216,7 @@ public class Section { return fileAttributes != null && fileAttributes.stream().anyMatch(attribute -> label.equals(attribute.getLabel()) && value.contains(attribute.getValue())); } + @SuppressWarnings("unused") @WhenCondition public boolean fileAttributeByIdEqualsIgnoreCase(@Argument(ArgumentType.FILE_ATTRIBUTE) String id, @Argument(ArgumentType.STRING) String value) { @@ -571,12 +574,12 @@ public class Section { @ThenAction @SuppressWarnings("unused") public void redactByRegExWithNewlines(@Argument(ArgumentType.REGEX) String pattern, - @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, - @Argument(ArgumentType.INTEGER) int group, - @Argument(ArgumentType.TYPE) String asType, - @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, - @Argument(ArgumentType.STRING) String reason, - @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { + @Argument(ArgumentType.BOOLEAN) boolean patternCaseInsensitive, + @Argument(ArgumentType.INTEGER) int group, + @Argument(ArgumentType.TYPE) String asType, + @Argument(ArgumentType.RULE_NUMBER) int ruleNumber, + @Argument(ArgumentType.STRING) String reason, + @Argument(ArgumentType.LEGAL_BASIS) String legalBasis) { redactByRegExWithNewlines(pattern, patternCaseInsensitive, group, asType, ruleNumber, reason, legalBasis, true, false); } @@ -665,7 +668,8 @@ public class Section { legalBasis, true, skipRemoveEntitiesContainedInLarger, - sortedResult, false); + sortedResult, + false); } @@ -697,11 +701,11 @@ public class Section { legalBasis, true, skipRemoveEntitiesContainedInLarger, - sortedResult, ignoreTables); + sortedResult, + ignoreTables); } - @ThenAction @SuppressWarnings("unused") public void redactBetween(@Argument(ArgumentType.STRING) String start, @@ -729,7 +733,8 @@ public class Section { legalBasis, true, skipRemoveEntitiesContainedInLarger, - sortedResult, false); + sortedResult, + false); } @@ -772,7 +777,8 @@ public class Section { legalBasis, true, skipRemoveEntitiesContainedInLarger, - sortedResult, false); + sortedResult, + false); } } @@ -787,19 +793,7 @@ public class Section { @Argument(ArgumentType.BOOLEAN) boolean redactEverywhere, @Argument(ArgumentType.STRING) String reason) { - redactBetween(start, - stop, - false, - false, - asType, - ruleNumber, - redactEverywhere, - false, - reason, - null, - false, - false, - false, false); + redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, false, reason, null, false, false, false, false); } @@ -813,19 +807,7 @@ public class Section { @Argument(ArgumentType.BOOLEAN) boolean excludeHeadLine, @Argument(ArgumentType.STRING) String reason) { - redactBetween(start, - stop, - false, - false, - asType, - ruleNumber, - redactEverywhere, - excludeHeadLine, - reason, - null, - false, - false, - false, false); + redactBetween(start, stop, false, false, asType, ruleNumber, redactEverywhere, excludeHeadLine, reason, null, false, false, false, false); } @@ -1293,14 +1275,14 @@ public class Section { public Set findEntities(String value, - String asType, - boolean caseInsensitive, - boolean redacted, - int ruleNumber, - String reason, - String legalBasis, - Engine engine, - boolean asRecommendation) { + String asType, + boolean caseInsensitive, + boolean redacted, + int ruleNumber, + String reason, + String legalBasis, + Engine engine, + boolean asRecommendation) { String text = caseInsensitive ? searchText.toLowerCase() : searchText; Set found = EntitySearchUtils.findEntities(text, @@ -1445,7 +1427,8 @@ public class Section { if (StringUtils.isNotBlank(stringOffset.getValue())) { var trimmedOffsetString = stringOffset.trim(); Set found = findEntities(trimmedOffsetString.getValue(), asType, false, true, ruleNumber, reason, legalBasis, Engine.RULE, false).stream() - .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()).collect(Collectors.toSet()); + .filter(f -> !onlyExactMatch || f.getStart() == trimmedOffsetString.getStart() && f.getEnd() == trimmedOffsetString.getEnd()) + .collect(Collectors.toSet()); found.forEach(f -> f.setSkipRemoveEntitiesContainedInLarger(skipRemoveEntitiesContainedInLarger)); EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); @@ -1459,7 +1442,15 @@ public class Section { } - private void redactByRegExWithNewlines(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) { + private void redactByRegExWithNewlines(String pattern, + boolean patternCaseInsensitive, + int group, + String asType, + int ruleNumber, + String reason, + String legalBasis, + boolean redaction, + boolean skipRemoveEntitiesContainedInLarger) { Pattern compiledPattern = Patterns.getCompiledMultilinePattern(pattern, patternCaseInsensitive); @@ -1476,7 +1467,15 @@ public class Section { } - private void redactByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, String reason, String legalBasis, boolean redaction, boolean skipRemoveEntitiesContainedInLarger) { + private void redactByRegEx(String pattern, + boolean patternCaseInsensitive, + int group, + String asType, + int ruleNumber, + String reason, + String legalBasis, + boolean redaction, + boolean skipRemoveEntitiesContainedInLarger) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -1524,8 +1523,7 @@ public class Section { boolean sortedResult, boolean ignoreTables) { - - if(isInTable && ignoreTables){ + if (isInTable && ignoreTables) { return; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java index d3f3b7d6..0488942e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/TenantDictionary.java @@ -3,6 +3,8 @@ package com.iqser.red.service.redaction.v1.server.redaction.model; import java.util.HashMap; import java.util.Map; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation; + import lombok.Data; import lombok.NoArgsConstructor; @@ -12,4 +14,5 @@ public class TenantDictionary { private final Map dictionariesByDossierTemplate = new HashMap<>(); private final Map dictionariesByDossier = new HashMap<>(); + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java similarity index 99% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java index 746a3a63..bbdb44fc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Dictionary.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import static java.lang.String.format; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java index 8a4d6c09..a715246a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryEntries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryEntries.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import java.util.HashSet; import java.util.Set; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java index 29a71403..961636cb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrement.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrement.java @@ -1,10 +1,10 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; + +import java.util.Set; import lombok.AllArgsConstructor; import lombok.Data; -import java.util.Set; - @Data @AllArgsConstructor public class DictionaryIncrement { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java similarity index 94% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java index ba762ed5..c50ed785 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryIncrementValue.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryIncrementValue.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java index 00b413d9..2a5424ee 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryModel.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryModel.java @@ -1,16 +1,15 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; - -import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; - -import lombok.AllArgsConstructor; -import lombok.Data; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import java.io.Serializable; import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; + +import lombok.AllArgsConstructor; +import lombok.Data; + @Data @AllArgsConstructor public class DictionaryModel implements Serializable { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java index be8a4134..7d084784 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryRepresentation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryRepresentation.java @@ -1,12 +1,12 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; - -import lombok.Data; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import lombok.Data; + @Data public class DictionaryRepresentation { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java similarity index 95% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java index 3017bd91..e1f46683 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/DictionaryVersion.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/DictionaryVersion.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.model; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java similarity index 97% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java index e5a87eed..15d4071d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/SearchImplementation.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/SearchImplementation.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.redaction.utils; +package com.iqser.red.service.redaction.v1.server.redaction.model.dictionary; import java.util.ArrayList; import java.util.Collection; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java index f90750e3..160d513c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java @@ -24,14 +24,14 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.DictionaryEntry; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryEntries; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryRepresentation; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.model.TenantDictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryEntries; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryRepresentation; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import feign.FeignException; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index b1e2dae6..200400fa 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -26,9 +26,9 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.Section; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; @@ -66,7 +66,7 @@ public class DroolsExecutionService { kieSession.setGlobal("dictionary", dictionary); document.getEntities().forEach(kieSession::insert); - document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); + document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode())); document.getPages().forEach(kieSession::insert); fileAttributes.forEach(kieSession::insert); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index c3f0537b..45dcc4b9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -9,6 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRedactionEntry; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.section.SectionArea; @@ -18,12 +19,11 @@ import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSeque import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle; import io.micrometer.core.annotation.Timed; import lombok.RequiredArgsConstructor; @@ -37,6 +37,7 @@ public class ManualRedactionSurroundingTextService { private final RedactionStorageService redactionStorageService; private final RedactionServiceSettings redactionServiceSettings; + @Timed("redactmanager_surroundingTextAnalysis") public AnalyzeResult addSurroundingText(String dossierId, String fileId, ManualRedactions manualRedactions) { @@ -87,9 +88,18 @@ public class ManualRedactionSurroundingTextService { } if (sectionText.getCellStarts() != null && !sectionText.getCellStarts().isEmpty()) { - SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, sectionText.getCellStarts(), redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); + SurroundingWordsService.addSurroundingText(Set.of(correctEntity), + sectionText.getSearchableText(), + null, + sectionText.getCellStarts(), + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); } else { - SurroundingWordsService.addSurroundingText(Set.of(correctEntity), sectionText.getSearchableText(), null, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); + SurroundingWordsService.addSurroundingText(Set.of(correctEntity), + sectionText.getSearchableText(), + null, + redactionServiceSettings.getSurroundingWordsOffsetWindow(), + redactionServiceSettings.getNumberOfSurroundingWords()); } return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java index 2bba58b1..726c9eb9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/SurroundingWordsService.java @@ -3,9 +3,9 @@ package com.iqser.red.service.redaction.v1.server.redaction.service; import java.util.List; import java.util.Set; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; import io.micrometer.core.annotation.Timed; import lombok.experimental.UtilityClass; @@ -15,7 +15,6 @@ import lombok.extern.slf4j.Slf4j; @UtilityClass public class SurroundingWordsService { - @Timed("redactmanager_addSurroundingText") public void addSurroundingText(Set entities, SearchableText searchableText, Dictionary dictionary, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) { @@ -38,7 +37,12 @@ public class SurroundingWordsService { @Timed("redactmanager_addSurroundingTextTables") - public void addSurroundingText(Set entities, SearchableText searchableText, Dictionary dictionary, List cellstarts, int surroundingWordsOffsetWindow, int numberOfSurroundingWords) { + public void addSurroundingText(Set entities, + SearchableText searchableText, + Dictionary dictionary, + List cellstarts, + int surroundingWordsOffsetWindow, + int numberOfSurroundingWords) { if (entities.isEmpty()) { return; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java index b8eecf05..0e226e96 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/AnalyzeService.java @@ -39,10 +39,10 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.exception.RedactionException; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import com.iqser.red.service.redaction.v1.server.redaction.service.ImportedRedactionService; @@ -86,8 +86,6 @@ public class AnalyzeService { ImportedRedactionService importedRedactionService; SectionFinder sectionFinder; DocumentGraphFactory documentGraphFactory; - DocumentDataMapper documentDataMapper; - DocumentGraphMapper documentGraphMapper; FunctionTimerValues redactmanagerAnalyzePagewiseValues; @@ -133,7 +131,7 @@ public class AnalyzeService { sectionText.getSectionAreas()))); log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); - redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, documentDataMapper.toDocumentData(documentGraph)); + redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.TEXT, DocumentDataMapper.toDocumentData(documentGraph)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SIMPLIFIED_TEXT, convert(text)); redactionStorageService.storeObject(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), FileType.SECTION_GRID, classifiedDoc.getSectionGrid()); @@ -154,7 +152,7 @@ public class AnalyzeService { long startTime = System.currentTimeMillis(); RedactionLog redactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); - DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); // not yet ready for reanalysis if (redactionLog == null || documentGraph == null || documentGraph.getNumberOfPages() == 0) { @@ -208,7 +206,7 @@ public class AnalyzeService { public AnalyzeResult analyze(AnalyzeRequest analyzeRequest) { long startTime = System.currentTimeMillis(); - DocumentGraph documentGraph = documentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); + DocumentGraph documentGraph = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId())); NerEntities nerEntities; if (redactionServiceSettings.isNerServiceEnabled()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java index 8ccb7cbd..820fae83 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/analyze/SectionFinder.java @@ -21,11 +21,11 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlo import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrement; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryIncrementValue; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.RedRectangle2D; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrement; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryIncrementValue; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import io.micrometer.core.annotation.Timed; import lombok.AccessLevel; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java index 0cf2d378..70d88e6f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityFinder.java @@ -11,12 +11,12 @@ import org.springframework.stereotype.Component; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.Entities; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils; import com.iqser.red.service.redaction.v1.server.redaction.utils.FindEntityDetails; import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index 7f47d849..482c3bdd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -13,9 +13,9 @@ import com.iqser.red.service.redaction.v1.server.client.model.NerEntities; import com.iqser.red.service.redaction.v1.server.document.graph.DocumentGraph; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNode; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.service.DroolsExecutionService; import lombok.AccessLevel; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java index 0b1a9659..75ba741b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/EntitySearchUtils.java @@ -13,13 +13,14 @@ import java.util.stream.Collectors; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.Entity; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; import com.iqser.red.service.redaction.v1.server.redaction.model.Image; import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java index ce8723a4..df6e68de 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/PdfDraw.java @@ -34,7 +34,7 @@ public class PdfDraw { public static void drawDocumentGraph(PDDocument document, DocumentGraph documentGraph) { - documentGraph.getTableOfContents().streamEntriesInOrder().forEach(entry -> drawNode(document, entry)); + documentGraph.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> drawNode(document, entry)); } @@ -132,7 +132,8 @@ public class PdfDraw { private static Options buildStandardOptionsForNodes(TableOfContents.Entry entry) { - return Options.builder().stroke(true).strokeColor(switch (entry.type()) { + return Options.builder().stroke(true).strokeColor(switch (entry.getType()) { + case DOCUMENT -> Color.LIGHT_GRAY; case HEADER, FOOTER -> Color.GREEN; case PARAGRAPH -> Color.BLUE; case HEADLINE -> Color.RED; @@ -146,9 +147,9 @@ public class PdfDraw { private static void drawBBoxAndLabelAndNumberOnPage(PDDocument document, TableOfContents.Entry entry, Options options) { - Map rectanglesPerPage = entry.node().getBBox(); + Map rectanglesPerPage = entry.getNode().getBBox(); rectanglesPerPage.forEach((page, rectangle2D) -> { - if (entry.type() == NodeType.SECTION) { + if (entry.getType() == NodeType.SECTION) { rectangle2D = RectangleTransformations.pad(rectangle2D, 10, 10); } drawRectangle2DList(document, page.getNumber(), List.of(rectangle2D), options); @@ -159,7 +160,7 @@ public class PdfDraw { private static String buildString(TableOfContents.Entry entry) { - return entry.node().getNumberOnPage() + ": " + entry.tocId() + ": " + entry.type().toString(); + return entry.getNode().getNumberOnPage() + ": " + entry.getTocId() + ": " + entry.getType().toString(); } } \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java index 86a3be53..deda5ec0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AbstractTestWithDictionaries.java @@ -65,10 +65,10 @@ import lombok.SneakyThrows; @Import(AbstractTestWithDictionaries.TestConfiguration.class) public class AbstractTestWithDictionaries { - protected static final String RULES = loadFromClassPath("drools/rules.drl"); + protected static final String RULES = loadFromClassPath("drools/testRules.drl"); protected static final String RULES_PATH = "drools/rules.drl"; protected static final String ENTITY_RULES_PATH = "drools/entity_rules.drl"; - protected static final String MERGE_ENTITY_RULES_PATH = "drools/merge_entity_rules.drl"; + protected static final String MERGE_ENTITY_RULES_PATH = "drools/testRules.drl"; private static final String VERTEBRATE = "vertebrate"; private static final String ADDRESS = "CBI_address"; private static final String AUTHOR = "CBI_author"; @@ -518,7 +518,7 @@ public class AbstractTestWithDictionaries { KieFileSystem kieFileSystem = kieServices.newKieFileSystem(); kieFileSystem.write(ResourceFactory.newClassPathResource(MERGE_ENTITY_RULES_PATH, "UTF-8")); - kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8")); + //kieFileSystem.write(ResourceFactory.newClassPathResource(ENTITY_RULES_PATH, "UTF-8")); KieRepository kieRepository = kieServices.getRepository(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java index fed7ce46..a78f9cc1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DictionaryServiceTest.java @@ -28,7 +28,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; import com.iqser.red.service.redaction.v1.server.client.DictionaryClient; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryVersion; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index e8a471a1..89922fbd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -28,12 +28,12 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SectionNod import com.iqser.red.service.redaction.v1.server.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService; import com.iqser.red.service.redaction.v1.server.redaction.service.RedactionLogCreatorService; -import com.iqser.red.service.redaction.v1.server.redaction.utils.SearchImplementation; import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw; @@ -98,7 +98,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { long serializationStart = System.currentTimeMillis(); document.getEntities().forEach(kieSession::insert); - document.getTableOfContents().streamEntriesInOrder().forEach(entry -> kieSession.insert(entry.node())); + document.getTableOfContents().streamAllEntriesInOrder().forEach(entry -> kieSession.insert(entry.getNode())); document.getPages().forEach(kieSession::insert); System.out.printf("Object serialization and kieSession insertion took %d ms\n", System.currentTimeMillis() - serializationStart); @@ -140,7 +140,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { durationMillis = ((float) (System.currentTimeMillis() - start)); System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n); - SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).node(); + SemanticNode paragraph = document.getTableOfContents().getEntryById(List.of(8, 1)).getNode(); start = System.currentTimeMillis(); for (int i = 0; i < n; i++) { paragraph.buildTextBlock(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java index 7e640ce8..53b305f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/FileSystemBackedStorageService.java @@ -1,5 +1,7 @@ package com.iqser.red.service.redaction.v1.server; +import static java.io.File.createTempFile; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -15,7 +17,6 @@ import org.springframework.core.io.InputStreamResource; import com.fasterxml.jackson.databind.ObjectMapper; import com.iqser.red.commons.jackson.ObjectMapperFactory; -import com.iqser.red.service.redaction.v1.server.tableextraction.utils.FileUtils; import com.iqser.red.storage.commons.exception.StorageObjectDoesNotExist; import com.iqser.red.storage.commons.service.StorageService; @@ -62,7 +63,7 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows public void storeJSONObject(String tenantId, String objectId, T any) { - File tempFile = FileUtils.createTempFile("test", ".tmp"); + File tempFile = createTempFile("test", ".tmp"); getMapper().writeValue(new FileOutputStream(tempFile), any); dataMap.put(objectId, tempFile); } @@ -101,7 +102,7 @@ public class FileSystemBackedStorageService implements StorageService { @SneakyThrows public void storeObject(String tenantId, String objectId, InputStream stream) { - File tempFile = FileUtils.createTempFile("test", ".tmp"); + File tempFile = createTempFile("test", ".tmp"); try (var fileOutputStream = new FileOutputStream(tempFile)) { IOUtils.copy(stream, fileOutputStream); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 607050f1..9738ef90 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -365,7 +365,7 @@ public class RedactionIntegrationTest { @Test public void titleExtraction() throws IOException { - AnalyzeRequest request = prepareStorage("files/new/RotateTestFile.pdf"); + AnalyzeRequest request = prepareStorage("files/new/crafted document.pdf"); analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); AnalyzeResult result = analyzeService.analyze(request); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java index 02101654..251bef57 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/annotate/AnnotationService.java @@ -143,6 +143,7 @@ public class AnnotationService { annotation.setTitlePopup(redactionLogEntry.getId()); annotation.setAnnotationName(redactionLogEntry.getId()); annotation.setColor(new PDColor(redactionLogEntry.getColor(), PDDeviceRGB.INSTANCE)); + annotation.setNoRotate(false); annotations.add(annotation); if (redactionLogEntry.getComments() != null) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java new file mode 100644 index 00000000..989a0806 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/BuildDocumentGraphTest.java @@ -0,0 +1,61 @@ +package com.iqser.red.service.redaction.v1.server.document.graph; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; + +import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; +import com.iqser.red.service.redaction.v1.server.classification.model.Document; +import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; +import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; +import com.iqser.red.service.redaction.v1.server.segmentation.ImageService; +import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; + +import lombok.SneakyThrows; + +public class BuildDocumentGraphTest extends AbstractTestWithDictionaries { + + @Autowired + private DocumentGraphFactory documentGraphFactory; + + @Autowired + private PdfSegmentationService segmentationService; + + @Autowired + private ImageService imageService; + + + @Test + public void buildMetolachlor() { + + DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); + assertEquals(221, documentGraph.getPages().size()); + assertEquals(220, documentGraph.getPages().stream().filter(page -> page.getHeader().hasText()).count()); + assertEquals(0, documentGraph.getPages().stream().filter(page -> page.getFooter().hasText()).count()); + } + + + @SneakyThrows + protected DocumentGraph buildGraph(String filename) { + + if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) { + prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json"); + } else { + prepareStorage(filename + ".pdf"); + } + ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + + try (InputStream inputStream = fileResource.getInputStream()) { + Map> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID); + Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages); + return documentGraphFactory.buildDocumentGraph(classifiedDoc); + } + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionTest.java similarity index 69% rename from redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java rename to redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionTest.java index d39f93bb..4d95b030 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphEntityInsertionTest.java @@ -4,18 +4,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.wildfly.common.Assert.assertTrue; -import java.io.InputStream; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; -import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; -import com.iqser.red.service.redaction.v1.server.classification.model.Document; import com.iqser.red.service.redaction.v1.server.document.graph.entity.EntityNode; -import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.HeadlineNode; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.document.graph.nodes.PageNode; @@ -26,25 +20,37 @@ import com.iqser.red.service.redaction.v1.server.document.graph.nodes.TableNode; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType; -import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage; -import com.iqser.red.service.redaction.v1.server.segmentation.ImageService; -import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; -import lombok.SneakyThrows; - -public class DocumentGraphTest extends AbstractTestWithDictionaries { - - @Autowired - private DocumentGraphFactory documentGraphFactory; - - @Autowired - private PdfSegmentationService segmentationService; +public class DocumentGraphEntityInsertionTest extends BuildDocumentGraphTest { @Autowired private EntityCreationService entityCreationService; - @Autowired - private ImageService imageService; + + @Test + public void assertCollectAllEntitiesWorks() { + + DocumentGraph documentGraph = buildGraph("files/new/crafted document"); + createAndInsertEntity(documentGraph, "Clarissa"); + createAndInsertEntity(documentGraph, "Lastname"); + createAndInsertEntity(documentGraph, "David Ksenia"); + createAndInsertEntity(documentGraph, "Michael N."); + createAndInsertEntity(documentGraph, "Page-Footer"); + createAndInsertEntity(documentGraph, "CTL/with dictionary entry 1234 with Slash"); + assertEquals(6, documentGraph.getEntities().size()); + } + + + private EntityNode createAndInsertEntity(DocumentGraph documentGraph, String searchTerm) { + + int start = documentGraph.getTextBlock().indexOf(searchTerm); + assert start != -1; + + Boundary boundary = new Boundary(start, start + searchTerm.length()); + EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY); + entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents()); + return entityNode; + } @Test @@ -52,11 +58,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/new/crafted document"); String searchTerm = "Clarissa"; - int start = documentGraph.getTextBlock().indexOf(searchTerm); - assert start != -1; - - Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm); assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("’s Donut ←", entityNode.getTextAfter()); @@ -67,7 +69,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } @@ -76,11 +78,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/new/crafted document"); String searchTerm = "Rule 39:"; - int start = documentGraph.getTextBlock().indexOf(searchTerm); - assert start != -1; - - Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm); assertEquals("", entityNode.getTextBefore()); assertEquals(" Purity Hint", entityNode.getTextAfter()); @@ -90,7 +88,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } @@ -99,11 +97,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/new/crafted document"); String searchTerm = "1998"; - int start = documentGraph.getTextBlock().indexOf(searchTerm); - assert start != -1; - - Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm); assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextAfter()); @@ -113,7 +107,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assertEquals(15, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } @@ -134,9 +128,9 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/new/crafted document"); TableNode table = (TableNode) documentGraph.getTableOfContents()// - .streamEntriesInOrder()// - .filter(entry -> entry.type().equals(NodeType.TABLE))// - .map(TableOfContents.Entry::node)// + .streamAllEntriesInOrder()// + .filter(entry -> entry.getType().equals(NodeType.TABLE))// + .map(TableOfContents.Entry::getNode)// .findFirst().orElseThrow(); assertEquals(5, table.getNumberOfCols()); assertEquals(4, table.getNumberOfRows()); @@ -164,10 +158,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); TableNode table = (TableNode) documentGraph.getTableOfContents() - .streamEntriesInOrder() - .filter(entry -> entry.node().getPages().stream().anyMatch(page -> page.getNumber() == 22)) - .filter(entry -> entry.type().equals(NodeType.TABLE)) - .map(TableOfContents.Entry::node) + .streamAllEntriesInOrder() + .filter(entry -> entry.getNode().getPages().stream().anyMatch(page -> page.getNumber() == 22)) + .filter(entry -> entry.getType().equals(NodeType.TABLE)) + .map(TableOfContents.Entry::getNode) .findFirst() .orElseThrow(); assertEquals(5, table.getNumberOfCols()); @@ -185,11 +179,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); String searchTerm = "Cucurbit"; - int start = documentGraph.getTextBlock().indexOf(searchTerm); - assert start != -1; - - Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals(", Group 9;", entityNode.getTextAfter()); @@ -197,9 +187,10 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assertEquals(searchTerm, entityNode.getValue()); assertEquals(2, entityNode.getIntersectingNodes().size()); assertEquals(5, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); + assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 10)); assertInstanceOf(ParagraphNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } @@ -214,7 +205,8 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY); + entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents()); assertEquals("2.6.1 Summary of ", entityNode.getTextBefore()); assertEquals(" and excretion in", entityNode.getTextAfter()); @@ -222,10 +214,11 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); assertEquals(searchTerm, entityNode.getValue()); assertEquals(2, entityNode.getIntersectingNodes().size()); - assertEquals(6, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); + assertEquals(4, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); + assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 33)); assertInstanceOf(HeadlineNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } @@ -234,11 +227,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { DocumentGraph documentGraph = buildGraph("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"); String searchTerm = "N-deacetylation product"; - int start = documentGraph.getTextBlock().indexOf(searchTerm); - assert start != -1; - - Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = createAndInsertEntity(documentGraph, searchTerm); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals(" of metabolite of", entityNode.getTextAfter()); @@ -246,43 +235,23 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assertEquals(3, entityNode.getIntersectingNodes().size()); assertEquals("2.7.2 Summary of metabolism, distribution and expression of residues in plants, poultry, lactating ruminants, pigs and fish ", entityNode.getDeepestFullyContainingNode().getHeadline().buildTextBlock().getSearchText()); - - // TODO: the number below is odd, due the cell (1, 2) containing 4 AtomicTextBlocks, even though it is an image. This should be fixed during structure analysis. - // In general, numberOnPage needs some more thought. - assertEquals(25, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); + assertTrue(entityNode.getPages().stream().allMatch(pageNode -> pageNode.getNumber() == 54)); + assertEquals(26, entityNode.getDeepestFullyContainingNode().getNumberOnPage()); assertInstanceOf(TableCellNode.class, entityNode.getDeepestFullyContainingNode()); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); } - @SneakyThrows - protected DocumentGraph buildGraph(String filename) { - - if (filename.equals("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06")) { - prepareStorage(filename + ".pdf", "files/cv_service_empty_response.json", filename + ".IMAGE_INFO.json"); - } else { - prepareStorage(filename + ".pdf"); - } - ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); - - try (InputStream inputStream = fileResource.getInputStream()) { - Map> pdfImages = imageService.convertImages(TEST_DOSSIER_ID, TEST_FILE_ID); - Document classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, inputStream, pdfImages); - return documentGraphFactory.buildDocumentGraph(classifiedDoc); - } - } - - - private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, int start, EntityNode entityNode) { + private static void assertSameOffsetInAllIntersectingNodes(String searchTerm, EntityNode entityNode) { List paragraphStart = entityNode.getIntersectingNodes().stream()// .map(SemanticNode::buildTextBlock)// .map(textBlock -> textBlock.indexOf(searchTerm))// .toList(); - paragraphStart.forEach(nodeStart -> assertEquals(start, nodeStart)); + paragraphStart.forEach(nodeStart -> assertEquals(entityNode.getBoundary().start(), nodeStart)); } @@ -293,15 +262,17 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = EntityNode.initialEntityNode(boundary, "123", EntityType.ENTITY); + entityCreationService.addEntityToGraph(entityNode, documentGraph.getTableOfContents()); PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); assertEquals(entityNode.getValue(), searchTerm); assertTrue(pageNode.getEntities().contains(entityNode)); assertTrue(documentGraph.getPages().stream().filter(page -> page != pageNode).noneMatch(page -> page.getEntities().contains(entityNode))); assertTrue(entityNode.getPages().contains(pageNode)); - assertSameOffsetInAllIntersectingNodes(searchTerm, start, entityNode); + assertSameOffsetInAllIntersectingNodes(searchTerm, entityNode); assertTrue(entityNode.getIntersectingNodes().stream().allMatch(node -> node.getEntities().contains(entityNode))); } } + diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java index 3cb8b888..6bb9fde2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphMappingTest.java @@ -1,37 +1,19 @@ package com.iqser.red.service.redaction.v1.server.document.graph; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.ClassPathResource; -import com.iqser.red.service.redaction.v1.server.AbstractTestWithDictionaries; import com.iqser.red.service.redaction.v1.server.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.document.data.AtomicTextBlockData; import com.iqser.red.service.redaction.v1.server.document.data.DocumentData; import com.iqser.red.service.redaction.v1.server.document.data.PageData; import com.iqser.red.service.redaction.v1.server.document.data.TableOfContentsData; -import com.iqser.red.service.redaction.v1.server.document.graph.factory.DocumentGraphFactory; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentDataMapper; import com.iqser.red.service.redaction.v1.server.document.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.multitenancy.TenantContext; -import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationService; import lombok.SneakyThrows; -public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { - - @Autowired - private DocumentGraphFactory documentGraphFactory; - - @Autowired - private PdfSegmentationService segmentationService; - - @Autowired - private DocumentDataMapper documentDataMapper; - - @Autowired - private DocumentGraphMapper documentGraphMapper; - +public class DocumentGraphMappingTest extends BuildDocumentGraphTest { @Test @SneakyThrows @@ -39,12 +21,9 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { String filename = "files/new/crafted document"; - prepareStorage(filename + ".pdf"); - ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); + DocumentGraph document = buildGraph(filename); + DocumentData documentData = DocumentDataMapper.toDocumentData(document); - var classifiedDoc = segmentationService.parseDocument(TEST_DOSSIER_ID, TEST_FILE_ID, fileResource.getInputStream(), null); - DocumentGraph document = documentGraphFactory.buildDocumentGraph(classifiedDoc); - DocumentData documentData = documentDataMapper.toDocumentData(document); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_PAGES" + ".json", documentData.getPages()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_TEXT" + ".json", documentData.getAtomicTextBlocks()); storageService.storeJSONObject(TenantContext.getTenantId(), filename + "_POSITIONS" + ".json", documentData.getAtomicPositionBlocks()); @@ -57,7 +36,13 @@ public class DocumentGraphMappingTest extends AbstractTestWithDictionaries { AtomicPositionBlockData[].class); TableOfContentsData tableOfContentsData = storageService.readJSONObject(TenantContext.getTenantId(), filename + "_STRUCTURE" + ".json", TableOfContentsData.class); - DocumentGraph newDocumentGraph = documentGraphMapper.toDocumentGraph(documentData); + DocumentData documentData2 = DocumentData.builder() + .pages(pageData) + .tableOfContents(tableOfContentsData) + .atomicTextBlocks(atomicTextBlockData) + .atomicPositionBlocks(atomicPositionBlockData) + .build(); + DocumentGraph newDocumentGraph = DocumentGraphMapper.toDocumentGraph(documentData2); assert document.toString().equals(newDocumentGraph.toString()); assert document.getTableOfContents().toString().equals(newDocumentGraph.getTableOfContents().toString()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java index a2215a71..43653b7d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphVisualizationTest.java @@ -13,7 +13,7 @@ import com.iqser.red.service.redaction.v1.server.visualization.service.PdfDraw; import lombok.SneakyThrows; -public class DocumentGraphVisualizationTest extends DocumentGraphTest { +public class DocumentGraphVisualizationTest extends BuildDocumentGraphTest { @Test @SneakyThrows diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 495479fe..bc770eb4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -18,18 +18,21 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import java.util.Set import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService; -import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary; -import com.iqser.red.service.redaction.v1.server.redaction.model.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; global DocumentGraph document global EntityCreationService entityCreationService global Dictionary dictionary +// --------------------------------------- queries ------------------------------------------------------------------- query "getFileAttributes" $fileAttribute: FileAttribute() end +// --------------------------------------- merging rules ------------------------------------------------------------------- + rule "merge intersecting Entities of same type" salience 100 @@ -50,10 +53,10 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" when $falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE) - entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + $entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then - entity.removeFromGraph(); - retract(entity); + $entity.removeFromGraph(); + retract($entity); end rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" @@ -89,6 +92,8 @@ rule "remove Entity of lower rank, when equal boundaries and entityType" retract($lowerRank); end +// --------------------------------------- local dictionary search ------------------------------------------------------------------- + rule "run local dictionary search" agenda-group "LOCAL_DICTIONARY_ADDS" salience -999 @@ -108,11 +113,11 @@ rule "1: Redact CBI Authors (Non vertebrate study)" no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); - setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + setFields($entity, 1, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) end @@ -120,11 +125,11 @@ rule "2: Redact CBI Authors (Vertebrate study)" no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY) then $entity.setRedaction(true); - setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); + setFields($entity, 2, "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); update($entity) end @@ -132,7 +137,7 @@ rule "3: Don't redact CBI Address (Non vertebrate study)" no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY) then setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null); @@ -156,9 +161,9 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" when $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) then - EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); - setFields($entity, 5, "Genitive Author", null, Engine.RULE); - insert(entity); + EntityNode falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + setFields(falsePositive, 5, "Genitive Author", null, Engine.RULE); + insert(falsePositive); end @@ -189,7 +194,7 @@ rule "7: Add CBI_author with \"et al.\" Regex" rule "8: Add recommendation for Addresses in Test Organism sections" when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + //FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then Set entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); @@ -228,7 +233,7 @@ rule "10: Redacted PII Personal Identification Information (Non vertebrate study no-loop true when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY) then $entity.setRedaction(true); @@ -253,7 +258,7 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)" rule "12: Redact Emails by RegEx (Non vertebrate study)" when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") $section: SectionNode(containsString("@")) then Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); @@ -277,7 +282,7 @@ rule "14: Redact line after contact information (Non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $string: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -289,7 +294,7 @@ rule "14: Redact line after contact information (Non vertebrate study)" "Telephone number:", "Telephone No:", "Telephone:", - "Phone No:", + "Phone No.", "Phone:", "Fax number:", "Fax:", @@ -324,7 +329,7 @@ rule "15: Redact line after contact information (Vertebrate study)" "Telephone number:", "Telephone No:", "Telephone:", - "Phone No:", + "Phone No.", "Phone:", "Fax number:", "Fax:", @@ -345,7 +350,7 @@ rule "16: redact line between contact keywords" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + not FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Set entities = new HashSet<>();