Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7074' into 'master'

Browse Source 
RED-7074: Design Subsection section tree structure algorithm

Closes RED-7074

See merge request redactmanager/redaction-service!413
This commit is contained in:
Maverick Studer 2024-05-28 16:23:51 +02:00
commit 8a09d23b37
22 changed files with 1574 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1"
val layoutParserVersion = "0.116.0"
val layoutParserVersion = "0.131.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"

73
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/AbstractSemanticNode.java Normal file
View File

@ -0,0 +1,73 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public abstract class AbstractSemanticNode implements GenericSemanticNode {
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock textBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Override
public TextBlock getTextBlock() {
if (textBlock == null) {
textBlock = GenericSemanticNode.super.getTextBlock();
}
return textBlock;
}
@Override
public String toString() {
return treeId.toString() + ": " + getType() + ": " + this.getTextBlock().buildSummary();
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
}
}

64
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Document.java
View File

@ -3,7 +3,6 @@ package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import java.awt.geom.Rectangle2D;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -11,10 +10,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
@ -23,29 +19,22 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
/**
* Represents the entire document as a node within the document's semantic structure.
*/
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Document implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId = Collections.emptyList();
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Document extends AbstractSemanticNode {
Set<Page> pages;
DocumentTree documentTree;
Integer numberOfPages;
TextBlock textBlock;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
@Builder.Default
static final SectionIdentifier sectionIdentifier = SectionIdentifier.document();
@ -57,20 +46,27 @@ public class Document implements GenericSemanticNode {
}
public TextBlock getTextBlock() {
/**
* Gets the sections of the document as a list.
*
* @return A list of all sections within the document.
*/
public List<Section> getAllSections() {
if (textBlock == null) {
textBlock = GenericSemanticNode.super.getTextBlock();
}
return textBlock;
return streamAllSubNodesOfType(NodeType.SECTION).map(node -> (Section) node)
.collect(Collectors.toList());
}
/**
* Gets the main sections of the document as a list.
*
* @return A list of main sections within the document.
* @return A list of main sections within the document
* @deprecated This method is marked for removal.
* Use {@link #streamChildrenOfType(NodeType)} instead,
* or {@link #getChildrenOfTypeSectionOrSuperSection()} which returns children of type SECTION as well as SUPER_SECTION.
*/
@Deprecated(forRemoval = true)
public List<Section> getMainSections() {
return streamChildrenOfType(NodeType.SECTION).map(node -> (Section) node)
@ -78,6 +74,18 @@ public class Document implements GenericSemanticNode {
}
/**
* Gets the direct children of type SECTION or SUPER_SECTION of the document as a list of SemanticNode objects.
*
* @return A list of all children of type SECTION or SUPER_SECTION.
*/
public List<SemanticNode> getChildrenOfTypeSectionOrSuperSection() {
return streamChildren().filter(semanticNode -> semanticNode.getType().equals(NodeType.SECTION) || semanticNode.getType().equals(NodeType.SUPER_SECTION))
.toList();
}
/**
* Streams all terminal (leaf) text blocks within the document in their natural order.
*
@ -85,7 +93,15 @@ public class Document implements GenericSemanticNode {
*/
public Stream<TextBlock> streamTerminalTextBlocksInOrder() {
return streamAllNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getTextBlock);
return streamAllNodes().filter(SemanticNode::isLeaf)
.map(SemanticNode::getTextBlock);
}
@Override
public List<Integer> getTreeId() {
return Collections.emptyList();
}
@ -119,7 +135,7 @@ public class Document implements GenericSemanticNode {
*/
private Stream<SemanticNode> streamAllNodes() {
return documentTree.allEntriesInOrder()
return getDocumentTree().allEntriesInOrder()
.map(DocumentTree.Entry::getNode);
}

3
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/DuplicatedParagraph.java
View File

@ -20,7 +20,8 @@ public class DuplicatedParagraph extends Paragraph {
@Override
public TextBlock getTextBlock() {
return Stream.of(leafTextBlock, unsortedLeafTextBlock).collect(new TextBlockCollector());
return Stream.of(leafTextBlock, unsortedLeafTextBlock)
.collect(new TextBlockCollector());
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Footer.java
View File

@ -18,31 +18,20 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Footer implements GenericSemanticNode {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Footer extends AbstractSemanticNode {
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Override
public NodeType getType() {
@ -75,17 +64,7 @@ public class Footer implements GenericSemanticNode {
@Override
public String toString() {
return treeId + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
return getTreeId() + ": " + NodeType.FOOTER + ": " + leafTextBlock.buildSummary();
}
}

31
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Header.java
View File

@ -18,34 +18,23 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
/**
* Represents the header part of a document page.
*/
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Header implements GenericSemanticNode {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Header extends AbstractSemanticNode {
final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty();
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Override
public boolean isLeaf() {
@ -78,17 +67,7 @@ public class Header implements GenericSemanticNode {
@Override
public String toString() {
return treeId + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
return getTreeId() + ": " + NodeType.HEADER + ": " + leafTextBlock.buildSummary();
}
}

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Headline.java
View File

@ -19,32 +19,22 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
/**
* Represents a headline in a document.
*/
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Headline implements GenericSemanticNode {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Headline extends AbstractSemanticNode {
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
SectionIdentifier sectionIdentifier;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Override
public NodeType getType() {
@ -70,7 +60,7 @@ public class Headline implements GenericSemanticNode {
@Override
public String toString() {
return treeId + ": " + NodeType.HEADLINE + ": " + leafTextBlock.buildSummary();
return getTreeId() + ": " + NodeType.HEADLINE + ": " + leafTextBlock.buildSummary();
}
@ -81,16 +71,6 @@ public class Headline implements GenericSemanticNode {
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
}
@Override
public SectionIdentifier getSectionIdentifier() {

19
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java
View File

@ -27,24 +27,20 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
/**
*
Represents an image within the document.
*/
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@NoArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Image implements GenericSemanticNode, IEntity {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Image extends AbstractSemanticNode implements IEntity {
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
String id;
TextBlock leafTextBlock;
@ -61,11 +57,6 @@ public class Image implements GenericSemanticNode, IEntity {
Page page;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
@Override
public NodeType getType() {
@ -112,7 +103,7 @@ public class Image implements GenericSemanticNode, IEntity {
@Override
public String toString() {
return treeId + ": " + getValue() + " " + position;
return getTreeId() + ": " + getValue() + " " + position;
}

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/NodeType.java
View File

@ -5,6 +5,7 @@ import java.util.Locale;
public enum NodeType {
DOCUMENT,
SECTION,
SUPER_SECTION,
HEADLINE,
PARAGRAPH,
TABLE,

28
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Paragraph.java
View File

@ -26,23 +26,11 @@ import lombok.experimental.SuperBuilder;
@SuperBuilder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PROTECTED)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Paragraph implements GenericSemanticNode {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Paragraph extends AbstractSemanticNode {
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock leafTextBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Override
public NodeType getType() {
@ -68,17 +56,7 @@ public class Paragraph implements GenericSemanticNode {
@Override
public String toString() {
return treeId + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
return getTreeId() + ": " + NodeType.PARAGRAPH + ": " + leafTextBlock.buildSummary();
}
}

41
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Section.java
View File

@ -19,6 +19,7 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
/**
@ -26,25 +27,11 @@ import lombok.extern.slf4j.Slf4j;
*/
@Slf4j
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class Section implements GenericSemanticNode {
@EqualsAndHashCode.Include
List<Integer> treeId;
TextBlock textBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
Map<Page, Rectangle2D> bBoxCache;
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class Section extends AbstractSemanticNode {
@Override
@ -73,20 +60,11 @@ public class Section implements GenericSemanticNode {
}
@Override
public TextBlock getTextBlock() {
if (textBlock == null) {
textBlock = GenericSemanticNode.super.getTextBlock();
}
return textBlock;
}
@Override
public String toString() {
return treeId.toString() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
return getTreeId() + ": " + NodeType.SECTION + ": " + this.getTextBlock().buildSummary();
}
@ -123,13 +101,4 @@ public class Section implements GenericSemanticNode {
}
@Override
public Map<Page, Rectangle2D> getBBox() {
if (bBoxCache == null) {
bBoxCache = GenericSemanticNode.super.getBBox();
}
return bBoxCache;
}
}

90
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/SuperSection.java Normal file
View File

@ -0,0 +1,90 @@
package com.iqser.red.service.redaction.v1.server.model.document.nodes;
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
/**
* Represents a section within a document, encapsulating both its textual content and semantic structure.
*/
@Slf4j
@Data
@SuperBuilder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class SuperSection extends AbstractSemanticNode {
@Override
public NodeType getType() {
return NodeType.SUPER_SECTION;
}
/**
* Checks if this section contains any tables.
*
* @return True if the section contains at least one table, false otherwise.
*/
public boolean hasTables() {
return streamAllSubNodesOfType(NodeType.TABLE).findAny()
.isPresent();
}
@Override
public SectionIdentifier getSectionIdentifier() {
return getHeadline().getSectionIdentifier();
}
@Override
public String toString() {
return getTreeId() + ": " + NodeType.SUPER_SECTION + ": " + this.getTextBlock().buildSummary();
}
public Headline getHeadline() {
return streamChildrenOfType(NodeType.HEADLINE)//
.map(node -> (Headline) node)//
.findFirst()//
.orElseGet(() -> getParent().getHeadline());
}
/**
* Checks if any headline within this section or its sub-nodes contains a given string.
*
* @param value The string to search for within headlines, case-sensitive.
* @return True if at least one headline contains the specified string, false otherwise.
*/
public boolean anyHeadlineContainsString(String value) {
return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsString(value));
}
/**
* Checks if any headline within this section or its sub-nodes contains a given string, case-insensitive.
*
* @param value The string to search for within headlines, case-insensitive.
* @return True if at least one headline contains the specified string, false otherwise.
*/
public boolean anyHeadlineContainsStringIgnoreCase(String value) {
return streamAllSubNodesOfType(NodeType.HEADLINE).anyMatch(h -> h.containsStringIgnoreCase(value));
}
}

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/TableCell.java
View File

@ -19,22 +19,18 @@ import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults;
import lombok.experimental.SuperBuilder;
/**
* Represents a single table cell within a table.
*/
@Data
@Builder
@SuperBuilder
@AllArgsConstructor
@FieldDefaults(level = AccessLevel.PRIVATE)
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
public class TableCell implements GenericSemanticNode {
@EqualsAndHashCode(onlyExplicitlyIncluded = true, callSuper = true)
public class TableCell extends AbstractSemanticNode {
@Builder.Default
Set<LayoutEngine> engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM));
@EqualsAndHashCode.Include
List<Integer> treeId;
int row;
int col;
boolean header;
@ -45,10 +41,6 @@ public class TableCell implements GenericSemanticNode {
TextBlock textBlock;
DocumentTree documentTree;
@Builder.Default
Set<TextEntity> entities = new HashSet<>();
@Override
@ -93,7 +85,7 @@ public class TableCell implements GenericSemanticNode {
@Override
public String toString() {
return treeId + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
return getTreeId() + ": " + NodeType.TABLE_CELL + ": " + this.getTextBlock().buildSummary();
}
}

13
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/DocumentGraphMapper.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@ -7,12 +8,11 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentData;
import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.DuplicatedParagraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
@ -21,6 +21,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
@ -68,6 +69,7 @@ public class DocumentGraphMapper {
SemanticNode node = switch (entryData.getType()) {
case SECTION -> buildSection(context);
case SUPER_SECTION -> buildSuperSection(context);
case PARAGRAPH -> buildParagraph(context, entryData.getProperties());
case HEADLINE -> buildHeadline(context);
case HEADER -> buildHeader(context);
@ -118,7 +120,7 @@ public class DocumentGraphMapper {
private TableCell buildTableCell(Context context, Map<String, String> properties) {
TableCell.TableCellBuilder builder = TableCell.builder();
TableCell.TableCellBuilder<?, ?> builder = TableCell.builder();
PropertiesMapper.parseTableCellProperties(properties, builder);
return builder.documentTree(context.documentTree).build();
}
@ -150,6 +152,11 @@ public class DocumentGraphMapper {
}
private SuperSection buildSuperSection(Context context) {
return SuperSection.builder().documentTree(context.documentTree).build();
}
private Paragraph buildParagraph(Context context, Map<String, String> properties) {
if (PropertiesMapper.isDuplicateParagraph(properties)) {

22
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -315,7 +315,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
@Test
public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/crafted document.pdf");
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
@ -475,7 +475,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
int correctFound = 0;
loop:
for (EntityLogEntry entityLogEntry : entityLog.getEntityLogEntry()) {
for (Section section : documentGraph.getMainSections()) {
for (Section section : documentGraph.getAllSections()) {
if (entityLogEntry.getEntryType().equals(EntryType.IMAGE)) {
correctFound++;
continue loop;
@ -1254,9 +1254,11 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
@Test
@Disabled
// todo: fix me in RED-9257
public void signaturesAreRedactionAfterReanalyse() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
@ -1295,7 +1297,9 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
@Test
public void entityIsAppliedAfterRecateorize() throws IOException {
@Disabled
// todo: fix me in RED-9257
public void entityIsAppliedAfterRecategorize() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
@ -1310,6 +1314,14 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
analyzeService.analyze(request);
System.out.println("Finished analysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var initialAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
.findFirst()
.get();
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
@ -1333,7 +1345,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changedAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java
View File

@ -257,7 +257,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals(", Group 9;", textEntity.getTextAfter());
assertEquals("1.1.4 Evaluations carried out under other regulatory contexts ", textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(searchTerm, textEntity.getValue());
assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(5, textEntity.getIntersectingNodes().size());
assertEquals(5, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(textEntity.getPages()
.stream()
@ -287,7 +287,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("1 Statement of subject matter and purpose for which this report has been prepared and background information on the application ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertEquals(searchTerm, textEntity.getValue());
assertEquals(3, textEntity.getIntersectingNodes().size());
assertEquals(4, textEntity.getIntersectingNodes().size());
assertEquals(1, textEntity.getDeepestFullyContainingNode().getNumberOnPage());
assertTrue(textEntity.getPages()
.stream()
@ -308,7 +308,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", textEntity.getTextBefore());
assertEquals(" of metabolite of", textEntity.getTextAfter());
assertEquals(searchTerm, textEntity.getValue());
assertEquals(4, textEntity.getIntersectingNodes().size());
assertEquals(7, textEntity.getIntersectingNodes().size());
assertEquals("Table 2.7-1: List of substances and metabolites and related structural formula ",
textEntity.getDeepestFullyContainingNode().getHeadline().getTextBlock().getSearchText());
assertTrue(textEntity.getPages()

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -181,8 +181,8 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
float durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on document took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
Section section = document.getMainSections()
.get(8);
Section section = document.getAllSections().get(9);
start = System.currentTimeMillis();
for (int i = 0; i < n; i++) {
section.getTextBlock();
@ -190,7 +190,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
durationMillis = ((float) (System.currentTimeMillis() - start));
System.out.printf("%d calls of buildTextBlock() on section took %f s, average is %f ms\n", n, durationMillis / 1000, durationMillis / n);
SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(8, 1)).getNode();
SemanticNode paragraph = document.getDocumentTree().getEntryById(List.of(3, 2, 4)).getNode();
start = System.currentTimeMillis();
for (int i = 0; i < n; i++) {
paragraph.getTextBlock();

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java
View File

@ -97,11 +97,11 @@ public class TableTest extends BuildDocumentIntegrationTest {
viewerDocumentService.addVisualizationsOnPage(file,
file,
Visualizations.builder()
.layer(ENTITY_LAYER)
.visualizationsOnPages(visualizationsOnPage)
.layerVisibilityDefaultValue(true)
.build());
List.of(Visualizations.builder()
.layer(ENTITY_LAYER)
.visualizationsOnPages(visualizationsOnPage)
.layerVisibilityDefaultValue(true)
.build()));
}
}

16
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/SemanticNodeComparatorsTest.java
View File

@ -16,8 +16,8 @@ class SemanticNodeComparatorsTest {
@Test
public void testFirstSemanticNode() {
var node = new Section(List.of(0, 1), null, null, null, null, null);
var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
var node = Section.builder().treeId(List.of(0, 1)).build();
var otherNode = Section.builder().treeId(List.of(0, 2)).build();
List<SemanticNode> list = new ArrayList<>();
list.add(otherNode);
list.add(node);
@ -29,8 +29,8 @@ class SemanticNodeComparatorsTest {
@Test
public void testFirstSemanticNode2() {
var node = new Section(Collections.emptyList(), null, null, null, null, null);
var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
var node = Section.builder().treeId(Collections.emptyList()).build();
var otherNode = Section.builder().treeId(List.of(0, 2)).build();
List<SemanticNode> list = new ArrayList<>();
list.add(otherNode);
list.add(node);
@ -42,8 +42,8 @@ class SemanticNodeComparatorsTest {
@Test
public void testFirstSemanticNode3() {
var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
var otherNode = new Section(List.of(0, 2), null, null, null, null, null);
var node = Section.builder().treeId(List.of(1, 5, 8)).build();
var otherNode = Section.builder().treeId(List.of(0, 2)).build();
List<SemanticNode> list = new ArrayList<>();
list.add(otherNode);
list.add(node);
@ -55,8 +55,8 @@ class SemanticNodeComparatorsTest {
@Test
public void testFirstSemanticNode4() {
var node = new Section(List.of(1, 5, 8), null, null, null, null, null);
var otherNode = new Section(List.of(1, 5, 9), null, null, null, null, null);
var node = Section.builder().treeId(List.of(1, 5, 8)).build();
var otherNode = Section.builder().treeId(List.of(1, 5, 9)).build();
List<SemanticNode> list = new ArrayList<>();
list.add(otherNode);
list.add(node);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/utils/PdfVisualisationUtility.java
View File

@ -123,7 +123,7 @@ public class PdfVisualisationUtility {
case HEADER, FOOTER -> Color.GREEN;
case PARAGRAPH -> Color.BLUE;
case HEADLINE -> Color.RED;
case SECTION -> Color.BLACK;
case SUPER_SECTION, SECTION -> Color.BLACK;
case TABLE -> Color.ORANGE;
case TABLE_CELL -> Color.GRAY;
case IMAGE -> Color.MAGENTA;

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.pdf Normal file
View File

Binary file not shown.

1288
redaction-service-v1/redaction-service-server-v1/src/test/resources/ner_entities/S-Metolachlor_RAR_02_Volume_2_2018-09-06.NER_ENTITIES.json
View File

File diff suppressed because one or more lines are too long