diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java index 1b64dee2..f517a93a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/DocumentGraphFactory.java @@ -83,7 +83,7 @@ public class DocumentGraphFactory { List textBlocks = new ArrayList<>(textBlocksToMerge); textBlocks.add(originalTextBlock); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); + AtomicTextBlock textBlock = context.textBlockFactory.fromContext(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), node, context, page); List treeId = context.documentTree.createNewChildEntryAndReturnId(parentNode, node); node.setLeafTextBlock(textBlock); node.setTreeId(treeId); @@ -145,10 +145,7 @@ public class DocumentGraphFactory { Page page = context.getPage(textBlocks.get(0).getPage()); Footer footer = Footer.builder().documentTree(context.getDocumentTree()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), - footer, - context, - page); + AtomicTextBlock textBlock = context.textBlockFactory.fromContext(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), footer, context, page); List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(footer); footer.setTreeId(tocId); footer.setLeafTextBlock(textBlock); @@ -160,7 +157,7 @@ public class DocumentGraphFactory { Page page = context.getPage(textBlocks.get(0).getPage()); Header header = Header.builder().documentTree(context.getDocumentTree()).build(); - AtomicTextBlock textBlock = context.textBlockFactory.buildAtomicTextBlock(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page); + AtomicTextBlock textBlock = context.textBlockFactory.fromNumberOnPage(TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(textBlocks), header, 0, page); List tocId = context.getDocumentTree().createNewMainEntryAndReturnId(header); header.setTreeId(tocId); header.setLeafTextBlock(textBlock); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java index a1a3661e..064db8ad 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TableNodeFactory.java @@ -33,7 +33,11 @@ public class TableNodeFactory { Set pages = tablesToMerge.stream().map(AbstractPageBlock::getPage).map(context::getPage).collect(Collectors.toSet()); List> mergedRows = tablesToMerge.stream().map(TablePageBlock::getRows).flatMap(Collection::stream).toList(); - Table table = Table.builder().documentTree(context.getDocumentTree()).numberOfCols(mergedRows.isEmpty() ? 0 :mergedRows.get(0).size()).numberOfRows(mergedRows.size()).build(); + Table table = Table.builder() + .documentTree(context.getDocumentTree()) + .numberOfCols(mergedRows.isEmpty() ? 0 : mergedRows.get(0).size()) + .numberOfRows(mergedRows.size()) + .build(); pages.forEach(page -> addTableToPage(page, parentNode, table)); @@ -109,13 +113,13 @@ public class TableNodeFactory { if (cell.getTextBlocks().isEmpty()) { tableCell.setLeafTextBlock(context.getTextBlockFactory().emptyTextBlock(tableNode, context, page)); } else if (cell.getTextBlocks().size() == 1) { - textBlock = context.getTextBlockFactory().buildAtomicTextBlock(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page); + textBlock = context.getTextBlockFactory().fromContext(cell.getTextBlocks().get(0).getSequences(), tableCell, context, page); tableCell.setLeafTextBlock(textBlock); } else if (firstTextBlockIsHeadline(cell)) { SectionNodeFactory.addSection(tableCell, cell.getTextBlocks().stream().map(tb -> (AbstractPageBlock) tb).toList(), emptyList(), context); } else if (cellAreaIsSmallerThanPageAreaTimesThreshold(cell, page)) { List sequences = TextPositionOperations.mergeAndSortTextPositionSequenceByYThenX(cell.getTextBlocks()); - textBlock = context.getTextBlockFactory().buildAtomicTextBlock(sequences, tableCell, context, page); + textBlock = context.getTextBlockFactory().fromContext(sequences, tableCell, context, page); tableCell.setLeafTextBlock(textBlock); } else { cell.getTextBlocks().forEach(tb -> DocumentGraphFactory.addParagraphOrHeadline(tableCell, tb, context, emptyList())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java index a1eef095..12c0157f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/TextBlockFactory.java @@ -17,14 +17,14 @@ public class TextBlockFactory { long textBlockIdx; - public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) { + public AtomicTextBlock fromContext(List sequences, SemanticNode parent, DocumentGraphFactory.Context context, Page page) { Integer numberOnPage = context.getAndIncrementTextBlockNumberOnPage(page); - return buildAtomicTextBlock(sequences, parent, numberOnPage, page); + return fromNumberOnPage(sequences, parent, numberOnPage, page); } - public AtomicTextBlock buildAtomicTextBlock(List sequences, SemanticNode parent, Integer numberOnPage, Page page) { + public AtomicTextBlock fromNumberOnPage(List sequences, SemanticNode parent, Integer numberOnPage, Page page) { SearchTextWithTextPositionDto searchTextWithTextPositionDto = SearchTextWithTextPositionFactory.buildSearchTextToTextPositionModel(sequences); int offset = stringOffset; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java index 4a44e00a..b043e490 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java @@ -4,18 +4,42 @@ import java.util.Collections; import java.util.Objects; import java.util.Set; -public record MatchedRule(RuleIdentifier ruleIdentifier, String reason, String legalBasis, boolean applied, Set references) implements Comparable { +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.experimental.FieldDefaults; + +@Getter +@Builder +@AllArgsConstructor +@EqualsAndHashCode +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public final class MatchedRule implements Comparable { + + @Builder.Default + RuleIdentifier ruleIdentifier = RuleIdentifier.empty(); + @Builder.Default + String reason = ""; + @Builder.Default + String legalBasis = ""; + boolean applied; + boolean writeValueWithLineBreaks; + @Builder.Default + Set references = Collections.emptySet(); + public static MatchedRule empty() { - return new MatchedRule(RuleIdentifier.empty(), "", "", false, Collections.emptySet()); + return MatchedRule.builder().build(); } @Override public int compareTo(MatchedRule matchedRule) { - RuleIdentifier otherRuleIdentifier = matchedRule.ruleIdentifier(); + RuleIdentifier otherRuleIdentifier = matchedRule.getRuleIdentifier(); if (!Objects.equals(ruleIdentifier.type(), otherRuleIdentifier.type())) { if (Objects.equals(otherRuleIdentifier.type(), "MAN")) { return 1; @@ -24,10 +48,17 @@ public record MatchedRule(RuleIdentifier ruleIdentifier, String reason, String l return -1; } } - if (!Objects.equals(otherRuleIdentifier.unit(), ruleIdentifier().unit())) { + if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) { return otherRuleIdentifier.unit() - ruleIdentifier.unit(); } return otherRuleIdentifier.id() - ruleIdentifier.id(); } + + @Override + public String toString() { + + return "MatchedRule[" + "ruleIdentifier=" + ruleIdentifier + ", " + "reason=" + reason + ", " + "legalBasis=" + legalBasis + ", " + "applied=" + applied + ", " + "writeValueWithLineBreaks=" + writeValueWithLineBreaks + ", " + "references=" + references + ']'; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java new file mode 100644 index 00000000..3da9d1b8 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java @@ -0,0 +1,104 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity; + +import java.util.Collection; +import java.util.HashSet; +import java.util.PriorityQueue; +import java.util.Set; + +import lombok.NonNull; + +public interface MatchedRuleHolder { + + PriorityQueue getMatchedRuleList(); + + + default boolean isApplied() { + + return getMatchedRule().isApplied(); + } + + + default Set getReferences() { + + return getMatchedRule().getReferences(); + } + + + default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { + + if (legalBasis.isBlank() || legalBasis.isEmpty()) { + throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); + } + getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build()); + } + + + default void applyWithLineBreaks(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { + + if (legalBasis.isBlank() || legalBasis.isEmpty()) { + throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); + } + getMatchedRuleList().add(MatchedRule.builder() + .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) + .reason(reason) + .legalBasis(legalBasis) + .applied(true) + .writeValueWithLineBreaks(true) + .build()); + } + + + default void applyWithReferences(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis, Collection references) { + + if (legalBasis.isBlank() || legalBasis.isEmpty()) { + throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); + } + getMatchedRuleList().add(MatchedRule.builder() + .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) + .reason(reason) + .legalBasis(legalBasis) + .applied(true) + .references(new HashSet<>(references)) + .build()); + } + + + default void skip(@NonNull String ruleIdentifier, String reason) { + + getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build()); + } + + + default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection references) { + + getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build()); + } + + + default void addMatchedRule(MatchedRule matchedRule) { + + getMatchedRuleList().add(matchedRule); + } + + + default void addMatchedRules(Collection matchedRules) { + + getMatchedRuleList().addAll(matchedRules); + } + + + default int getMatchedRuleUnit() { + + return getMatchedRule().getRuleIdentifier().unit(); + } + + + default MatchedRule getMatchedRule() { + + if (getMatchedRuleList().isEmpty()) { + return MatchedRule.empty(); + } + return getMatchedRuleList().peek(); + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index 4fa0f78f..8933e854 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -2,7 +2,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.e import java.awt.geom.Rectangle2D; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedList; @@ -22,7 +21,6 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.NonNull; import lombok.experimental.FieldDefaults; @Data @@ -30,7 +28,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) @EqualsAndHashCode(onlyExplicitlyIncluded = true) -public class RedactionEntity { +public class RedactionEntity implements MatchedRuleHolder { // initial values @EqualsAndHashCode.Include @@ -73,18 +71,6 @@ public class RedactionEntity { } - public boolean isApplied() { - - return getMatchedRule().applied(); - } - - - public Set getReferences() { - - return getMatchedRule().references(); - } - - public boolean occursInNodeOfType(Class clazz) { return intersectingNodes.stream().anyMatch(clazz::isInstance); @@ -121,6 +107,12 @@ public class RedactionEntity { } + public String getValueWithLineBreaks() { + + return getDeepestFullyContainingNode().getTextBlock().subSequenceWithLineBreaks(getBoundary()); + } + + public void removeFromGraph() { intersectingNodes.forEach(node -> node.getEntities().remove(this)); @@ -133,63 +125,6 @@ public class RedactionEntity { } - public void apply(@NonNull String ruleIdentifier, String comment, @NonNull String legalBasis) { - - if (legalBasis.isBlank() || legalBasis.isEmpty()) { - throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); - } - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, legalBasis, true, Collections.emptySet())); - } - - - public void applyWithReferences(@NonNull String ruleIdentifier, String comment, @NonNull String legalBasis, Collection references) { - - if (legalBasis.isBlank() || legalBasis.isEmpty()) { - throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); - } - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, legalBasis, true, new HashSet<>(references))); - } - - - public void skip(@NonNull String ruleIdentifier, String comment) { - - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, "", false, Collections.emptySet())); - } - - - public void skipWithReferences(@NonNull String ruleIdentifier, String comment, Collection references) { - - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, "", false, new HashSet<>(references))); - } - - - public void addMatchedRule(MatchedRule matchedRule) { - - matchedRuleList.add(matchedRule); - } - - - public void addMatchedRules(Collection matchedRules) { - - matchedRuleList.addAll(matchedRules); - } - - - public int getMatchedRuleUnit() { - - return getMatchedRule().ruleIdentifier().unit(); - } - - - public MatchedRule getMatchedRule() { - - if (matchedRuleList.isEmpty()) { - return MatchedRule.empty(); - } - return matchedRuleList.peek(); - } - - public List getRedactionPositionsPerPage() { if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java index 1e129f1f..6007f411 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java @@ -12,7 +12,6 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java index 6dea4c7a..5419a070 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java @@ -6,7 +6,6 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import lombok.AccessLevel; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java index 2b7f1c98..57e89e12 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java @@ -6,7 +6,6 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import lombok.AccessLevel; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java index 83af32c2..8eae571b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java @@ -6,7 +6,6 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java index d9fad9e0..e0b1dddd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java @@ -1,7 +1,6 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes; import java.awt.geom.Rectangle2D; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -12,8 +11,8 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.MatchedRuleHolder; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RuleIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; @@ -23,7 +22,6 @@ import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import lombok.NonNull; import lombok.experimental.FieldDefaults; @Data @@ -31,7 +29,7 @@ import lombok.experimental.FieldDefaults; @AllArgsConstructor @NoArgsConstructor @FieldDefaults(level = AccessLevel.PRIVATE) -public class Image implements GenericSemanticNode { +public class Image implements GenericSemanticNode, MatchedRuleHolder { List treeId; String id; @@ -56,75 +54,6 @@ public class Image implements GenericSemanticNode { Set entities = new HashSet<>(); - public void apply(@NonNull String ruleIdentifier, String comment, @NonNull String legalBasis) { - - if (legalBasis.isBlank() || legalBasis.isEmpty()) { - throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); - } - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, legalBasis, true, Collections.emptySet())); - } - - - public void applyWithReferences(@NonNull String ruleIdentifier, String comment, @NonNull String legalBasis, Collection references) { - - if (legalBasis.isBlank() || legalBasis.isEmpty()) { - throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity"); - } - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, legalBasis, true, new HashSet<>(references))); - } - - - public void skip(@NonNull String ruleIdentifier, String comment) { - - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, "", false, Collections.emptySet())); - } - - - public void skipWithReferences(@NonNull String ruleIdentifier, String comment, Collection references) { - - matchedRuleList.add(new MatchedRule(RuleIdentifier.fromString(ruleIdentifier), comment, "", false, new HashSet<>(references))); - } - - - public void addMatchedRule(MatchedRule matchedRule) { - - matchedRuleList.add(matchedRule); - } - - - public void addMatchedRules(Collection matchedRules) { - - matchedRuleList.addAll(matchedRules); - } - - - public boolean isApplied() { - - return getMatchedRule().applied(); - } - - - public Set getReferences() { - - return getMatchedRule().references(); - } - - - public int getMatchedRuleUnit() { - - return getMatchedRule().ruleIdentifier().unit(); - } - - - public MatchedRule getMatchedRule() { - - if (matchedRuleList.isEmpty()) { - return MatchedRule.empty(); - } - return matchedRuleList.peek(); - } - - @Override public NodeType getType() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java index 0d70d987..04082a92 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java @@ -6,7 +6,6 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionIdentifier.java similarity index 99% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionIdentifier.java index d18d5fdf..b1455c2f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SectionIdentifier.java @@ -1,4 +1,4 @@ -package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers; +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes; import java.util.Collections; import java.util.LinkedList; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index 6d4bcf2f..22eff118 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -13,7 +13,6 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java index 7a7376ec..b5ebdead 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlock.java @@ -9,6 +9,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicPositionBlockData; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.AtomicTextBlockData; @@ -200,6 +202,38 @@ public class AtomicTextBlock implements TextBlock { } + @Override + public String subSequenceWithLineBreaks(Boundary boundary) { + + if (boundary.length() == 0 || !getBoundary().contains(boundary)) { + return ""; + } + + CharSequence subSequence = subSequence(boundary); + Set lbInBoundary = lineBreaks.stream().filter(boundary::contains).collect(Collectors.toSet()); + if (boundary.end() == getBoundary().end()) { + lbInBoundary.add(getBoundary().length()); + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < subSequence.length(); i++) { + char character = subSequence.charAt(i); + if (lbInBoundary.contains(i + 1)) { + // always plus one, due to the linebreaks being an exclusive end index + if (!Character.isWhitespace(character)) { + lbInBoundary.remove(i + 1); + lbInBoundary.add(i + 2); + sb.append(character); + continue; + } + sb.append("\n"); + } else { + sb.append(character); + } + } + return sb.toString(); + } + + private List getAllLineBreaksInBoundary(Boundary boundary) { return getLineBreaks().stream().map(linebreak -> linebreak + this.boundary.start()).filter(boundary::contains).toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java index 6fda9ad8..c560814e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/ConcatenatedTextBlock.java @@ -172,6 +172,34 @@ public class ConcatenatedTextBlock implements TextBlock { } + @Override + public String subSequenceWithLineBreaks(Boundary boundary) { + + if (boundary.length() == 0 || !getBoundary().contains(boundary)) { + return ""; + } + + List textBlocks = getAllAtomicTextBlocksPartiallyInStringBoundary(boundary); + + if (textBlocks.size() == 1) { + return textBlocks.get(0).subSequenceWithLineBreaks(boundary); + } + + StringBuilder sb = new StringBuilder(); + AtomicTextBlock firstTextBlock = textBlocks.get(0); + sb.append(firstTextBlock.subSequenceWithLineBreaks(new Boundary(boundary.start(), firstTextBlock.getBoundary().end()))); + + for (AtomicTextBlock textBlock : textBlocks.subList(1, textBlocks.size() - 1)) { + sb.append(textBlock.searchTextWithLineBreaks()); + } + + var lastTextBlock = textBlocks.get(textBlocks.size() - 1); + sb.append(lastTextBlock.subSequenceWithLineBreaks(new Boundary(lastTextBlock.getBoundary().start(), boundary.end()))); + + return sb.toString(); + } + + private Map> mergeEntityPositionsWithSamePageNode(Map> map1, Map> map2) { Map> mergedMap = new HashMap<>(map1); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java index bc8cc330..f757ae81 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/TextBlock.java @@ -42,9 +42,18 @@ public interface TextBlock extends CharSequence { Map> getPositionsPerPage(Boundary stringBoundary); + String subSequenceWithLineBreaks(Boundary boundary); + + int numberOfLines(); + default String searchTextWithLineBreaks() { + + return subSequenceWithLineBreaks(getBoundary()); + } + + default int indexOf(String searchTerm) { return indexOf(searchTerm, getBoundary().start()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 021e9de1..666bd9ef 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -23,6 +23,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Do import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell; @@ -155,16 +156,19 @@ public class EntityCreationService { .map(boundary -> byBoundary(boundary, type, entityType, node)); } - public Stream byRegexWithLinebreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) { - return byRegexWithLinebreaks(regexPattern, type, entityType, 0, node); + public Stream byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) { + + return byRegexWithLineBreaks(regexPattern, type, entityType, 0, node); } - public Stream byRegexWithLinebreaksIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) { - return byRegexWithLinebreaksIgnoreCase(regexPattern, type, entityType, 0, node); + public Stream byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) { + + return byRegexWithLineBreaksIgnoreCase(regexPattern, type, entityType, 0, node); } + public Stream byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { return byRegex(regexPattern, type, entityType, 0, node); @@ -177,17 +181,22 @@ public class EntityCreationService { } - public Stream byRegexWithLinebreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { + public Stream byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexWithLinebreaks(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByRegexWithLineBreaks(regexPattern, group, node.getTextBlock()) + .stream() + .map(boundary -> byBoundary(boundary, type, entityType, node)); } - public Stream byRegexWithLinebreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { + public Stream byRegexWithLineBreaksIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { - return RedactionSearchUtility.findBoundariesByRegexWithLinebreaksIgnoreCase(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); + return RedactionSearchUtility.findBoundariesByRegexWithLineBreaksIgnoreCase(regexPattern, group, node.getTextBlock()) + .stream() + .map(boundary -> byBoundary(boundary, type, entityType, node)); } + public Stream byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) { return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.getTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node)); @@ -208,6 +217,12 @@ public class EntityCreationService { } + public Stream bySemanticNodeParagraphsOnly(SemanticNode node, String type, EntityType entityType) { + + return node.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(semanticNode -> bySemanticNode(semanticNode, type, entityType)).filter(Optional::isPresent).map(Optional::get); + } + + public Optional bySemanticNode(SemanticNode node, String type, EntityType entityType) { Boundary boundary = node.getTextBlock().getBoundary(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java index 81bc9907..1ca46870 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java @@ -102,17 +102,17 @@ public class RedactionSearchUtility { } - public static List findBoundariesByRegexWithLinebreaks(String regexPattern, int group, TextBlock textBlock) { + public static List findBoundariesByRegexWithLineBreaks(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, false); - return getBoundariesByPatternWithLinebreaks(textBlock, group, pattern); + return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern); } - public static List findBoundariesByRegexWithLinebreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) { + public static List findBoundariesByRegexWithLineBreaksIgnoreCase(String regexPattern, int group, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledMultilinePattern(regexPattern, true); - return getBoundariesByPatternWithLinebreaks(textBlock, group, pattern); + return getBoundariesByPatternWithLineBreaks(textBlock, group, pattern); } @@ -134,21 +134,10 @@ public class RedactionSearchUtility { } - private static List getBoundariesByPatternWithLinebreaks(TextBlock textBlock, int group, Pattern pattern) { + private static List getBoundariesByPatternWithLineBreaks(TextBlock textBlock, int group, Pattern pattern) { - StringBuilder stringBuilder = new StringBuilder(); - textBlock.getAtomicTextBlocks().forEach(at -> { - if (at.numberOfLines() > 1) { - for (int i = 0; i < at.numberOfLines(); i++) { - stringBuilder.append(at.getLine(i)); - stringBuilder.setCharAt(stringBuilder.length() - 1, '\n'); - } - } else { - stringBuilder.append(at.getSearchText()).setCharAt(stringBuilder.length() - 1, '\n'); - } - }); - - Matcher matcher = pattern.matcher(stringBuilder.toString()); + String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks(); + Matcher matcher = pattern.matcher(searchTextWithLineBreaks); List boundaries = new LinkedList<>(); while (matcher.find()) { boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start())); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 33dba4bf..1c465566 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -85,9 +85,9 @@ public class RedactionLogCreatorService { return RedactionLogEntry.builder() .color(getColor(entity.getType(), dossierTemplateId, entity.isApplied())) - .reason(entity.getMatchedRule().reason()) - .legalBasis(entity.getMatchedRule().legalBasis()) - .value(entity.getValue()) + .reason(entity.getMatchedRule().getReason()) + .legalBasis(entity.getMatchedRule().getLegalBasis()) + .value(entity.getMatchedRule().isWriteValueWithLineBreaks() ? entity.getValueWithLineBreaks() : entity.getValue()) .type(entity.getType()) .redacted(entity.isApplied()) .isHint(isHint(entity.getType(), dossierTemplateId)) @@ -95,7 +95,7 @@ public class RedactionLogCreatorService { .isFalsePositive(entity.getEntityType().equals(EntityType.FALSE_POSITIVE) || entity.getEntityType().equals(EntityType.FALSE_RECOMMENDATION)) .section(entity.getDeepestFullyContainingNode().toString()) .sectionNumber(sectionNumber) - .matchedRule(entity.getMatchedRule().ruleIdentifier().toString()) + .matchedRule(entity.getMatchedRule().getRuleIdentifier().toString()) .isDictionaryEntry(entity.isDictionaryEntry()) .textAfter(entity.getTextAfter()) .textBefore(entity.getTextBefore()) @@ -117,9 +117,9 @@ public class RedactionLogCreatorService { .isImage(true) .type(imageType) .redacted(image.isApplied()) - .reason(image.getMatchedRule().reason()) - .legalBasis(image.getMatchedRule().legalBasis()) - .matchedRule(image.getMatchedRule().ruleIdentifier().toString()) + .reason(image.getMatchedRule().getReason()) + .legalBasis(image.getMatchedRule().getLegalBasis()) + .matchedRule(image.getMatchedRule().getRuleIdentifier().toString()) .isHint(dictionaryService.isHint(image.getImageType().toString(), dossierTemplateId)) .isDictionaryEntry(false) .isRecommendation(false) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java index cbefb1dc..bb0431b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -13,7 +13,6 @@ import java.util.Set; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; @@ -35,7 +34,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemp import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.storage.commons.StorageAutoConfiguration; import com.iqser.red.storage.commons.service.StorageService; @@ -48,9 +46,6 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); - @Autowired - private EntityCreationService entityCreationService; - @Configuration @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/RedactionEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/RedactionEntityTest.java index 24be60ad..2c261855 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/RedactionEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/RedactionEntityTest.java @@ -20,7 +20,7 @@ public class RedactionEntityTest { entity.skip("CBI.3.0", ""); entity.skip("CBI.4.1", ""); entity.skip("CBI.4.0", ""); - assertThat(entity.getMatchedRule().ruleIdentifier().toString()).isEqualTo("CBI.4.1"); + assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.4.1"); assertThat(entity.getMatchedRuleUnit()).isEqualTo(4); } @@ -34,7 +34,7 @@ public class RedactionEntityTest { entity.skip("CBI.3.0", ""); entity.skip("CBI.4.1", ""); entity.skip("CBI.4.0", ""); - assertThat(entity.getMatchedRule().ruleIdentifier().toString()).isEqualTo("MAN.2.0"); + assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("MAN.2.0"); assertThat(entity.getMatchedRuleUnit()).isEqualTo(2); } @@ -59,7 +59,7 @@ public class RedactionEntityTest { entity.apply("CBI.0.0", "", ""); }); entity.skip("CBI.2.0", ""); - assertThat(entity.getMatchedRule().ruleIdentifier().toString()).isEqualTo("CBI.2.0"); + assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.2.0"); assertThat(entity.getMatchedRuleUnit()).isEqualTo(2); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java index 443cf69a..1d0b04d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java @@ -138,7 +138,7 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati assertFalse(entity.getIntersectingNodes().isEmpty()); assertEquals(1, entity.getPages().size()); assertEquals("David Ksenia", entity.getValue()); - assertEquals("Something", entity.getMatchedRule().legalBasis()); + assertEquals("Something", entity.getMatchedRule().getLegalBasis()); assertEquals(initialId, entity.getRedactionPositionsPerPage().get(0).getId()); assertFalse(entity.isRemoved()); assertTrue(entity.isSkipRemoveEntitiesContainedInLarger()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java index 4e1871db..36d89b62 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java @@ -5,6 +5,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionIdentifier; + class SectionIdentifierTest { @Test diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlockTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlockTest.java new file mode 100644 index 00000000..76bb80e1 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/textblock/AtomicTextBlockTest.java @@ -0,0 +1,93 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; + +class AtomicTextBlockTest { + + @Test + void subSequenceWithLineBreaks1() { + + String searchText = "1234 6789 "; + var atb = AtomicTextBlock.builder().searchText(searchText).lineBreaks(List.of(5, 7)).boundary(new Boundary(0, searchText.length())).build(); + String searchTextWithLineBreaks = atb.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n", searchTextWithLineBreaks); + } + + + @Test + void subSequenceWithLineBreaks2() { + + String searchText = "1234 6789 "; + var atb = AtomicTextBlock.builder().searchText(searchText).lineBreaks(List.of(5, 7, 8, 9)).boundary(new Boundary(0, searchText.length())).build(); + String searchTextWithLineBreaks = atb.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n", searchTextWithLineBreaks); + } + + + @Test + void subSequenceWithLineBreaks3() { + + String searchText = "1234 6789 1234 "; + var atb = AtomicTextBlock.builder().searchText(searchText).lineBreaks(List.of(5, 7)).boundary(new Boundary(0, searchText.length())).build(); + String searchTextWithLineBreaks = atb.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n1234\n", searchTextWithLineBreaks); + } + + + @Test + void subSequenceWithLineBreaks4() { + + String searchText = "1234 6789 1234 "; + var atb = AtomicTextBlock.builder().searchText(searchText).lineBreaks(List.of(5, 7)).boundary(new Boundary(0, searchText.length())).build(); + var textBlock = new ConcatenatedTextBlock(List.of(atb)); + String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n1234\n", searchTextWithLineBreaks); + } + + + @Test + void subSequenceWithLineBreaks5() { + + String searchText1 = "1234 6789 "; + String searchText2 = "1234 "; + var atb1 = AtomicTextBlock.builder().searchText(searchText1).lineBreaks(List.of(5, 7)).boundary(new Boundary(0, searchText1.length())).build(); + var atb2 = AtomicTextBlock.builder() + .searchText(searchText2) + .lineBreaks(List.of()) + .boundary(new Boundary(searchText1.length(), searchText1.length() + searchText2.length())) + .build(); + var textBlock = new ConcatenatedTextBlock(List.of(atb1, atb2)); + String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n1234\n", searchTextWithLineBreaks); + } + + + @Test + void subSequenceWithLineBreaks6() { + + String searchText1 = "1234 6789 "; + String searchText2 = "1234 "; + String searchText3 = "1234 8475678900 "; + var atb1 = AtomicTextBlock.builder().searchText(searchText1).lineBreaks(List.of(5, 7)).boundary(new Boundary(0, searchText1.length())).build(); + var atb2 = AtomicTextBlock.builder() + .searchText(searchText2) + .lineBreaks(List.of()) + .boundary(new Boundary(searchText1.length(), searchText1.length() + searchText2.length())) + .build(); + var atb3 = AtomicTextBlock.builder() + .searchText(searchText3) + .lineBreaks(List.of(atb2.getBoundary().end() + 6)) + .boundary(new Boundary(atb2.getBoundary().end(), atb2.getBoundary().end() + searchText3.length())) + .build(); + var textBlock = new ConcatenatedTextBlock(List.of(atb1, atb2, atb3)); + String searchTextWithLineBreaks = textBlock.searchTextWithLineBreaks(); + assertEquals("1234\n6789\n1234\n1234 8475678900\n", searchTextWithLineBreaks); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index af82bc6a..c820ee21 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -667,6 +667,7 @@ rule "X.1.0: merge intersecting Entities of same type" retract($first); retract($second); insert(mergedEntity); + mergedEntity.getIntersectingNodes().forEach(node -> update(node)); end @@ -677,6 +678,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then + $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.removeFromGraph(); retract($entity) end @@ -726,6 +728,7 @@ rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENT $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) then + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); $lowerRank.removeFromGraph(); retract($lowerRank); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index e44ff68b..bd11ba30 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -14,6 +14,11 @@ import java.util.Optional; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Headline; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; @@ -308,7 +313,18 @@ rule "DOC.5.0: Strain" entity.apply("DOC.5.0", "Strain found.", "n-a"); }); end - +rule "DOC.6.0" + when + Headline(containsStringIgnoreCase("materials and methods"), $sectionIdentifierMaterials: getSectionIdentifier()) + Headline(containsStringIgnoreCase("controls"), getSectionIdentifier().isChildOf($sectionIdentifierMaterials), $sectionIdentifierControls: getSectionIdentifier()) + $headline: Headline(containsStringIgnoreCase("positive control substances"), getSectionIdentifier().isChildOf($sectionIdentifierControls)) + then + System.out.println($headline); + entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "irgendwas", EntityType.ENTITY) + .forEach(entity -> { + entity.apply("DOC.6.0", "positive control substance found", "n-a"); + }); + end //rule "DOC.7.0: study title by document structure" // when @@ -328,7 +344,7 @@ rule "DOC.7.0: study title" when $section: Section(isOnPage(1) && (containsString("Final Report") || containsString("SPL"))) then - entityCreationService.byRegexWithLinebreaks("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { + entityCreationService.byRegexWithLineBreaks("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { entity.apply("DOC.7.0", "Title found", "n-a"); }); entityCreationService.betweenStrings("TITLE", "DATA REQUIREMENT", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { @@ -591,7 +607,7 @@ rule "DOC.14.0: Dosages" entityCreationService.betweenStrings("Animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> { entity.apply("DOC.14.0", "Dosage found", "n-a"); }); - entityCreationService.byRegexWithLinebreaks("(?:\\.[\\s|\\n]|^.{5,20}\\n)([^\\.]{1,200}(?:animal|given|received)[^\\.]{1,200}dose\\s(?:levels?\\s)?(?:of|at)[^\\.]{1,200})(?:\\.[\\s|\\n|$])", "dosages", EntityType.ENTITY,1, $section).forEach(entity -> { + entityCreationService.byRegexWithLineBreaks("(?:\\.[\\s|\\n]|^.{5,20}\\n)([^\\.]{1,200}(?:animal|given|received)[^\\.]{1,200}dose\\s(?:levels?\\s)?(?:of|at)[^\\.]{1,200})(?:\\.[\\s|\\n|$])", "dosages", EntityType.ENTITY,1, $section).forEach(entity -> { entity.apply("DOC.14.0", "Dosage found", "n-a"); }); end