diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java index eabe4ecc..d162304d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/factory/SectionNodeFactory.java @@ -80,7 +80,7 @@ public class SectionNodeFactory { remainingBlocks.removeAll(alreadyMerged); if (abstractPageBlock instanceof TextPageBlock) { - List textBlocks = findTextBlocksWithSameClassificationAndAlignsY(abstractPageBlock, remainingBlocks); + List textBlocks = findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(abstractPageBlock, remainingBlocks); alreadyMerged.addAll(textBlocks); DocumentGraphFactory.addParagraphOrHeadline(section, (TextPageBlock) abstractPageBlock, context, textBlocks); } else if (abstractPageBlock instanceof TablePageBlock tablePageBlock) { @@ -162,14 +162,15 @@ public class SectionNodeFactory { } - private List findTextBlocksWithSameClassificationAndAlignsY(AbstractPageBlock atc, List pageBlocks) { + private List findTextBlocksWithSameClassificationAndAlignsYAndSameOrientation(AbstractPageBlock atc, List pageBlocks) { return pageBlocks.stream() - .filter(abstractTextContainer -> !abstractTextContainer.equals(atc)) - .filter(abstractTextContainer -> abstractTextContainer.getPage() == atc.getPage()) - .filter(abstractTextContainer -> abstractTextContainer instanceof TextPageBlock) - .filter(abstractTextContainer -> abstractTextContainer.intersectsY(atc)) - .map(abstractTextContainer -> (TextPageBlock) abstractTextContainer) + .filter(abstractPageBlock -> !abstractPageBlock.equals(atc)) + .filter(abstractPageBlock -> abstractPageBlock.getPage() == atc.getPage()) + .filter(abstractPageBlock -> abstractPageBlock.getOrientation().equals(atc.getOrientation())) + .filter(abstractPageBlock -> abstractPageBlock.intersectsY(atc)) + .filter(abstractPageBlock -> abstractPageBlock instanceof TextPageBlock) + .map(abstractPageBlock -> (TextPageBlock) abstractPageBlock) .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java index 719b2edb..1e129f1f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Document.java @@ -12,6 +12,7 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; @@ -35,6 +36,8 @@ public class Document implements GenericSemanticNode { TextBlock textBlock; @Builder.Default Set entities = new HashSet<>(); + @Builder.Default + static final SectionIdentifier sectionIdentifier = SectionIdentifier.document(); @Override @@ -79,6 +82,13 @@ public class Document implements GenericSemanticNode { } + @Override + public SectionIdentifier getSectionIdentifier() { + + return sectionIdentifier; + } + + @Override public Headline getHeadline() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java index 42688e85..6dea4c7a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Footer.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -23,6 +24,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Footer implements GenericSemanticNode { + @Builder.Default + final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty(); + List treeId; TextBlock leafTextBlock; @@ -55,6 +59,13 @@ public class Footer implements GenericSemanticNode { } + @Override + public SectionIdentifier getSectionIdentifier() { + + return sectionIdentifier; + } + + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java index 4b7c50d8..2b7f1c98 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Header.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import lombok.AccessLevel; @@ -23,6 +24,9 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) public class Header implements GenericSemanticNode { + @Builder.Default + final static SectionIdentifier sectionIdentifier = SectionIdentifier.empty(); + List treeId; TextBlock leafTextBlock; @@ -55,6 +59,13 @@ public class Header implements GenericSemanticNode { } + @Override + public SectionIdentifier getSectionIdentifier() { + + return sectionIdentifier; + } + + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java index a1507a77..83af32c2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Headline.java @@ -6,6 +6,7 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; @@ -26,6 +27,7 @@ public class Headline implements GenericSemanticNode { List treeId; TextBlock leafTextBlock; + SectionIdentifier sectionIdentifier; @EqualsAndHashCode.Exclude DocumentTree documentTree; @@ -70,12 +72,24 @@ public class Headline implements GenericSemanticNode { } + @Override + public SectionIdentifier getSectionIdentifier() { + + if (sectionIdentifier == null) { + sectionIdentifier = SectionIdentifier.fromSearchText(getTextBlock().getSearchText()); + } + return sectionIdentifier; + } + + public static Headline empty() { return Headline.builder().leafTextBlock(AtomicTextBlock.empty(-1L, 0, new Page(), -1, null)).build(); } - public boolean hasParagraphs(){ + + public boolean hasParagraphs() { + return getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java index 101b9987..0d70d987 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Section.java @@ -6,9 +6,9 @@ import java.util.Set; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlockCollector; -import java.util.stream.Stream; import lombok.AccessLevel; import lombok.AllArgsConstructor; @@ -49,6 +49,13 @@ public class Section implements GenericSemanticNode { } + @Override + public SectionIdentifier getSectionIdentifier() { + + return getHeadline().getSectionIdentifier(); + } + + @Override public TextBlock getTextBlock() { @@ -75,19 +82,22 @@ public class Section implements GenericSemanticNode { } - public boolean anyHeadlineContainsString(String value){ + public boolean anyHeadlineContainsString(String value) { + return streamChildrenOfType(NodeType.HEADLINE)// .map(node -> (Headline) node).anyMatch(h -> h.containsString(value)); } - public boolean anyHeadlineContainsStringIgnoreCase(String value){ + public boolean anyHeadlineContainsStringIgnoreCase(String value) { + return streamChildrenOfType(NodeType.HEADLINE)// .map(node -> (Headline) node).anyMatch(h -> h.containsStringIgnoreCase(value)); } - public boolean hasParagraphs(){ + public boolean hasParagraphs() { + return streamAllSubNodesOfType(NodeType.PARAGRAPH).findFirst().isPresent(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index 3e18869b..6d4bcf2f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -13,6 +13,7 @@ import java.util.stream.Stream; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers.SectionIdentifier; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.AtomicTextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations; @@ -115,6 +116,17 @@ public interface SemanticNode { } + /** + * Returns a SectionIdentifier, such that it acts as a child of the first Headline associated with this SemanticNode. + * + * @return The SectionIdentifier from the first Headline. + */ + default SectionIdentifier getSectionIdentifier() { + + return SectionIdentifier.asChildOf(getHeadline().getSectionIdentifier()); + } + + /** * Checks if its TreeId has a length greater than zero. * diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java new file mode 100644 index 00000000..d18d5fdf --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifier.java @@ -0,0 +1,123 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.experimental.FieldDefaults; + +@AllArgsConstructor +@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) +public class SectionIdentifier { + + static Pattern numericalIdentifierPattern = Pattern.compile("^[\\s]?(\\d+)[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?[\\s.,;]?(\\d+)?"); + + private enum Format { + EMPTY, + NUMERICAL, + DOCUMENT + } + + Format format; + String identifierString; + List identifiers; + boolean asChild; + + + public static SectionIdentifier fromSearchText(String headline) { + + if (headline == null || headline.isEmpty() || headline.isBlank()) { + return SectionIdentifier.empty(); + } + + Matcher numericalIdentifierMatcher = numericalIdentifierPattern.matcher(headline); + if (numericalIdentifierMatcher.find()) { + return buildNumericalSectionIdentifier(headline, numericalIdentifierMatcher); + } + // more formats here + return SectionIdentifier.empty(); + } + + + public static SectionIdentifier asChildOf(SectionIdentifier sectionIdentifier) { + + return new SectionIdentifier(sectionIdentifier.format, sectionIdentifier.toString(), sectionIdentifier.identifiers, true); + } + + + public static SectionIdentifier document() { + + return new SectionIdentifier(Format.DOCUMENT, "document", Collections.emptyList(), false); + } + + + public static SectionIdentifier empty() { + + return new SectionIdentifier(Format.EMPTY, "empty", Collections.emptyList(), false); + } + + + private static SectionIdentifier buildNumericalSectionIdentifier(String headline, Matcher numericalIdentifierMatcher) { + + String identifierString = headline.substring(numericalIdentifierMatcher.start(), numericalIdentifierMatcher.end()); + List identifiers = new LinkedList<>(); + for (int i = 1; i <= 4; i++) { + String numericalIdentifier = numericalIdentifierMatcher.group(i); + if (numericalIdentifier == null || numericalIdentifier.equals("0") || numericalIdentifier.isEmpty() || numericalIdentifier.isBlank()) { + break; + } + identifiers.add(Integer.parseInt(numericalIdentifier.trim())); + } + return new SectionIdentifier(Format.NUMERICAL, identifierString, identifiers.stream().toList(), false); + } + + + /** + * Determines if the current section is the parent of the given section. + * + * @param sectionIdentifier The section identifier to compare against. + * @return true if the current section is the parent of the given section, false otherwise. + */ + public boolean isParentOf(SectionIdentifier sectionIdentifier) { + + if (this.format.equals(Format.EMPTY)) { + return false; + } + if (this.format.equals(Format.DOCUMENT)) { + return true; + } + if (!this.format.equals(sectionIdentifier.format)) { + return false; + } + if (this.identifiers.size() >= sectionIdentifier.identifiers.size() && !(this.identifiers.size() == sectionIdentifier.identifiers.size() && sectionIdentifier.asChild)) { + return false; + } + for (int i = 0; i < this.identifiers.size(); i++) { + if (!this.identifiers.get(i).equals(sectionIdentifier.identifiers.get(i))) { + return false; + } + } + return true; + } + + + public boolean isChildOf(SectionIdentifier sectionIdentifier) { + + if (this.format.equals(Format.DOCUMENT) || this.format.equals(Format.EMPTY)) { + return false; + } + return sectionIdentifier.isParentOf(this); + } + + + @Override + public String toString() { + + return identifierString; + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java index fdc3639f..4c101e57 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformations.java @@ -116,7 +116,7 @@ public class RectangleTransformations { @Override public BiConsumer accumulator() { - return (bb, rect) -> bb.addRectangle(rect.getMinX(), rect.getMinY(), rect.getMaxX(), rect.getMaxY()); + return BBox::addRectangle; } @@ -154,7 +154,12 @@ public class RectangleTransformations { Double upperRightY; - public void addRectangle(double lowerLeftX, double lowerLeftY, double upperRightX, double upperRightY) { + public void addRectangle(Rectangle2D rectangle2D) { + + double lowerLeftX = Math.min(rectangle2D.getMinX(), rectangle2D.getMaxX()); + double lowerLeftY = Math.min(rectangle2D.getMinY(), rectangle2D.getMaxY()); + double upperRightX = Math.max(rectangle2D.getMinX(), rectangle2D.getMaxX()); + double upperRightY = Math.max(rectangle2D.getMinY(), rectangle2D.getMaxY()); if (this.lowerLeftX == null) { this.lowerLeftX = lowerLeftX; diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java index 4cc67431..9e8d1f9c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java @@ -98,13 +98,16 @@ public class AnalyzeService { pdfImages = imageServiceResponseAdapter.convertImages(analyzeRequest.getDossierId(), analyzeRequest.getFileId()); } + log.info("parse document for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); classifiedDoc = pdfSegmentationService.parseDocument(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), storedObjectStream, pdfImages); } catch (Exception e) { throw new RedactionException(e); } + log.info("Build Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); Document document = DocumentGraphFactory.buildDocumentGraph(classifiedDoc); + log.info("Build section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); SectionGrid sectionGrid = sectionGridCreatorService.createSectionGrid(document); log.info("Store document graph, text, simplified text, and section grid for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java index 0b4b2f2a..b6fafe06 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java @@ -23,7 +23,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; import com.iqser.red.service.redaction.v1.server.client.RulesClient; import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException; -import com.knecon.fforesight.tenantcommons.TenantContext; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; @@ -91,6 +90,7 @@ public class DroolsExecutionService { kieSession.setGlobal("dictionary", dictionary); kieSession.setGlobal("nerEntitiesAdapter", nerEntitiesAdapter); + kieSession.insert(document); document.getEntities().forEach(kieSession::insert); sectionsToAnalyze.forEach(kieSession::insert); sectionsToAnalyze.stream().flatMap(SemanticNode::streamAllSubNodes).forEach(kieSession::insert); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java index 961e3bc9..33dba4bf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionLogCreatorService.java @@ -80,10 +80,7 @@ public class RedactionLogCreatorService { private RedactionLogEntry createRedactionLogEntry(RedactionEntity entity, String dossierTemplateId) { Set referenceIds = new HashSet<>(); - entity.getReferences() - .stream() - .filter(redactionEntity -> !redactionEntity.isRemoved() && !redactionEntity.isIgnored()) - .forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId()))); + entity.getReferences().stream().filter(RedactionEntity::isActive).forEach(ref -> ref.getRedactionPositionsPerPage().forEach(pos -> referenceIds.add(pos.getId()))); int sectionNumber = entity.getDeepestFullyContainingNode().getTreeId().isEmpty() ? 0 : entity.getDeepestFullyContainingNode().getTreeId().get(0); return RedactionLogEntry.builder() diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java new file mode 100644 index 00000000..cbefb1dc --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java @@ -0,0 +1,169 @@ +package com.iqser.red.service.redaction.v1.server; + +import static org.mockito.Mockito.when; +import static org.wildfly.common.Assert.assertFalse; +import static org.wildfly.common.Assert.assertTrue; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; +import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeResult; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive; +import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type; +import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; +import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; +import com.iqser.red.storage.commons.StorageAutoConfiguration; +import com.iqser.red.storage.commons.service.StorageService; +import com.knecon.fforesight.tenantcommons.TenantContext; + +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@Import(RedactionIntegrationTest.RedactionIntegrationTestConfiguration.class) +public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest { + + private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); + + @Autowired + private EntityCreationService entityCreationService; + + @Configuration + @EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class}) + @ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)}) + public static class RedactionIntegrationTestConfiguration { + + @Bean + @Primary + public StorageService inmemoryStorage() { + + return new FileSystemBackedStorageService(); + } + + } + + + @BeforeEach + public void stubClients() { + + TenantContext.setTenantId("redaction"); + + when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(JSONPrimitive.of(RULES)); + + loadDictionaryForTest(); + loadTypeForTest(); + loadNerForTest(); + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossierTemplate(TEST_DOSSIER_TEMPLATE_ID, false)).thenReturn(getTypeResponse()); + + when(dictionaryClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L); + when(dictionaryClient.getAllTypesForDossier(TEST_DOSSIER_ID, false)).thenReturn(List.of(Type.builder() + .id(DOSSIER_REDACTIONS_INDICATOR + ":" + TEST_DOSSIER_TEMPLATE_ID) + .type(DOSSIER_REDACTIONS_INDICATOR) + .dossierTemplateId(TEST_DOSSIER_ID) + .hexColor("#ffe187") + .isHint(hintTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isCaseInsensitive(caseInSensitiveMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .isRecommendation(recommendationTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .rank(rankTypeMap.get(DOSSIER_REDACTIONS_INDICATOR)) + .build())); + + mockDictionaryCalls(null); + + when(dictionaryClient.getColors(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(colors); + } + + + @Test + public void acceptanceTests() throws IOException { + + AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf"); + System.out.println("Start Full integration test"); + analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId())); + System.out.println("Finished structure analysis"); + AnalyzeResult result = analyzeService.analyze(request); + System.out.println("Finished analysis"); + var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var publishedInformationEntry1 = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals("published_information")) + .filter(entry -> entry.getValue().equals("Oxford University Press")) + .findFirst() + .orElseThrow(); + + var asyaLyon1 = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals("CBI_author")) + .filter(entry -> entry.getValue().equals("Asya Lyon")) + .filter(entry -> entry.getSectionNumber() == publishedInformationEntry1.getSectionNumber()) + .findFirst() + .orElseThrow(); + + assertFalse(asyaLyon1.isRedacted()); + + var idRemoval = IdRemoval.builder() + .requestDate(OffsetDateTime.now()) + .annotationId(publishedInformationEntry1.getId()) + .status(AnnotationStatus.APPROVED) + .fileId(TEST_FILE_ID) + .build(); + + var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build(); + request.setManualRedactions(manualRedactions); + analyzeService.reanalyze(request); + + redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); + + var publishedInformationEntry2 = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals("published_information")) + .filter(entry -> entry.getValue().equals("Oxford University Press")) + .findFirst() + .orElseThrow(); + + var asyaLyon2 = redactionLog.getRedactionLogEntry() + .stream() + .filter(entry -> entry.getType().equals("CBI_author")) + .filter(entry -> entry.getValue().equals("Asya Lyon")) + .filter(entry -> entry.getSectionNumber() == publishedInformationEntry2.getSectionNumber()) + .findFirst() + .orElseThrow(); + + assertTrue(asyaLyon2.isRedacted()); + + AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); + + String outputFileName = OsUtils.getTemporaryDirectory() + "/AcceptanceTest.pdf"; + + try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { + fileOutputStream.write(annotateResponse.getDocument()); + } + + } + +} diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java new file mode 100644 index 00000000..4e1871db --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/sectionidentifiers/SectionIdentifierTest.java @@ -0,0 +1,56 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.sectionidentifiers; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class SectionIdentifierTest { + + @Test + public void testParentOf() { + + var headline = SectionIdentifier.fromSearchText("1 Did you ever hear the tragedy of Darth Plagueis The Wise?"); + var headline1 = SectionIdentifier.fromSearchText("1.0 I thought not. It’s not a story the Jedi would tell you."); + var headline2 = SectionIdentifier.fromSearchText("1.1 It’s a Sith legend. Darth Plagueis was a Dark Lord of the Sith, "); + var headline3 = SectionIdentifier.fromSearchText("1.2.3 so powerful and so wise he could use the Force to influence the midichlorians to create life…"); + var headline4 = SectionIdentifier.fromSearchText("1.2.3.4 He had such a knowledge of the dark side that he could even keep the ones he cared about from dying."); + var headline5 = SectionIdentifier.fromSearchText("1.2.3.4.5 The dark side of the Force is a pathway to many abilities some consider to be unnatural."); + var headline6 = SectionIdentifier.fromSearchText("2.0 He became so powerful…"); + var headline7 = SectionIdentifier.fromSearchText("10000.0 the only thing he was afraid of was losing his power,"); + var headline8 = SectionIdentifier.fromSearchText("A.0 which eventually, of course, he did."); + var headline9 = SectionIdentifier.fromSearchText("Unfortunately, he taught his apprentice everything he knew, then his apprentice killed him in his sleep."); + var headline10 = SectionIdentifier.fromSearchText("2.1.2 Ironic."); + var headline11 = SectionIdentifier.fromSearchText("2.He could save others from death,"); + var headline12 = SectionIdentifier.fromSearchText(" 2. but not himself."); + + var paragraph1 = SectionIdentifier.asChildOf(headline); + assertTrue(paragraph1.isChildOf(headline)); + assertTrue(headline.isParentOf(paragraph1)); + assertFalse(paragraph1.isParentOf(headline)); + + assertFalse(headline.isParentOf(headline1)); + assertTrue(headline.isParentOf(headline2)); + assertTrue(headline.isParentOf(headline3)); + assertTrue(headline.isParentOf(headline4)); + assertTrue(headline.isParentOf(headline5)); + assertTrue(headline1.isParentOf(headline2)); + assertFalse(headline1.isParentOf(headline1)); + assertTrue(headline3.isParentOf(headline4)); + assertFalse(headline4.isParentOf(headline5)); + assertFalse(headline2.isParentOf(headline3)); + assertFalse(headline2.isParentOf(headline4)); + assertTrue(headline1.isParentOf(headline3)); + assertTrue(headline1.isParentOf(headline4)); + assertFalse(headline1.isParentOf(headline6)); + assertFalse(headline1.isParentOf(headline7)); + assertFalse(headline8.isParentOf(headline1)); + assertFalse(headline8.isParentOf(headline2)); + assertFalse(headline8.isParentOf(headline3)); + assertFalse(headline8.isParentOf(headline4)); + assertFalse(headline9.isParentOf(headline9)); + assertTrue(headline10.isChildOf(headline11)); + assertTrue(headline10.isChildOf(headline12)); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformationsTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformationsTest.java new file mode 100644 index 00000000..6bc9603f --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RectangleTransformationsTest.java @@ -0,0 +1,90 @@ +package com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.awt.geom.Rectangle2D; +import java.util.List; + +import org.junit.jupiter.api.Test; + +class RectangleTransformationsTest { + + @Test + public void testRectangle2DBBox() { + + var r1 = new Rectangle2D.Double(0, 0, 1, 1); + var r2 = new Rectangle2D.Double(1, 1, 1, 1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2)); + assertEquals(0, result.getX()); + assertEquals(0, result.getY()); + assertEquals(2, result.getWidth()); + assertEquals(2, result.getHeight()); + } + + + @Test + public void testRectangle2DBBox2() { + + var r1 = new Rectangle2D.Double(0, 0, -1, -1); + var r2 = new Rectangle2D.Double(1, 1, 1, 1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2)); + assertEquals(-1, result.getX()); + assertEquals(-1, result.getY()); + assertEquals(3, result.getWidth()); + assertEquals(3, result.getHeight()); + } + + + @Test + public void testRectangle2DBBox3() { + + var r1 = new Rectangle2D.Double(0, 0, -1, -1); + var r2 = new Rectangle2D.Double(1, 1, 1, 1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1)); + assertEquals(-1, result.getX()); + assertEquals(-1, result.getY()); + assertEquals(3, result.getWidth()); + assertEquals(3, result.getHeight()); + } + + + @Test + public void testRectangle2DBBox4() { + + var r1 = new Rectangle2D.Double(2, 0, -1, -1); + var r2 = new Rectangle2D.Double(0, 2, 1, -1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1)); + assertEquals(0, result.getX()); + assertEquals(-1, result.getY()); + assertEquals(2, result.getWidth()); + assertEquals(3, result.getHeight()); + } + + + @Test + public void testRectangle2DBBox5() { + + var r1 = new Rectangle2D.Double(2, 0, -1, -1); + var r2 = new Rectangle2D.Double(0, 2, 1, -1); + var r3 = new Rectangle2D.Double(3, 2, 1, 1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r2, r1, r3)); + assertEquals(0, result.getX()); + assertEquals(-1, result.getY()); + assertEquals(4, result.getWidth()); + assertEquals(4, result.getHeight()); + } + + + @Test + public void testRectangle2DBBox6() { + + var r1 = new Rectangle2D.Double(0, 0, -1, -1); + var r2 = new Rectangle2D.Double(-1, -1, -1, -1); + var result = RectangleTransformations.rectangle2DBBox(List.of(r1, r2)); + assertEquals(-2, result.getX()); + assertEquals(-2, result.getY()); + assertEquals(2, result.getWidth()); + assertEquals(2, result.getHeight()); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt index 9b21c07e..737e1a40 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/CBI_author.txt @@ -143,6 +143,7 @@ Allen T. Allen T.R. Almeida A Almeida A. +Asya Lyon Almeida A.A. Almeida A.A.|Vassilieff I. Almeida|A.A.|Vassilieff|I. diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt index 66b5a466..798747f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt @@ -87,3 +87,4 @@ Toxicol Sci. Toxicol Sci. 1 Test Ignored Hint Published Information Workshop +Oxford University Press diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl new file mode 100644 index 00000000..af82bc6a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -0,0 +1,761 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Table; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService; +import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter; +import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities; +import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility; + +global Document document +global EntityCreationService entityCreationService +global ManualRedactionApplicationService manualRedactionApplicationService +global NerEntitiesAdapter nerEntitiesAdapter +global Dictionary dictionary + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//------------------------------------ Syngenta specific rules ------------------------------------ + +// Rule unit: SYN.1 +rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" + when + $section: Section(containsString("CT") || containsString("BL")) + then + /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */ + entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.skip("SYN.1.0", ""); + entity.addEngine(Engine.RULE); + insert(entity); + }); + end + + +//------------------------------------ CBI rules ------------------------------------ + +// Rule unit: CBI.0 +rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", dictionaryEntry) + then + $entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_author", dictionaryEntry) + then + $entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: CBI.1 +rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", dictionaryEntry) + then + $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); + end + +rule "CBI.1.1: Redact CBI Address (Vertebrate Study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $entity: RedactionEntity(type == "CBI_address", dictionaryEntry) + then + $entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: CBI.2 +rule "CBI.2.0: Don't redact genitive CBI_author" + when + $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied()) + then + RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + falsePositive.skip("CBI.2.0", "Genitive Author found"); + insert(falsePositive); + end + + +// Rule unit: CBI.7 +rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables" + when + $section: Section(!hasTables(), + hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.skipWithReferences( + "CBI.7.0", + "Published Information found in section", + $section.getEntitiesOfType("published_information") + ); + }); + end + +rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row" + when + $table: Table(hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + then + $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address")) + .forEach(redactionEntity -> { + redactionEntity.skipWithReferences( + "CBI.7.1", + "Published Information found in row", + $table.getEntitiesOfTypeInSameRow("published_information", redactionEntity) + ); + }); + end + + +// Rule unit: CBI.9 +rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> { + redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + redactionEntity.addEngine(Engine.RULE); + insert(redactionEntity); + }); + end + +rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> { + redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + redactionEntity.addEngine(Engine.RULE); + insert(redactionEntity); + }); + end + + +// Rule unit: CBI.10 +rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)")) + then + $table.streamTableCellsWithHeader("Author(s)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> { + redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + redactionEntity.addEngine(Engine.RULE); + insert(redactionEntity); + }); + end + +rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> { + redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + redactionEntity.addEngine(Engine.RULE); + insert(redactionEntity); + }); + end + + +// Rule unit: CBI.11 +rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -1 + when + $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) + then + $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + end + + +// Rule unit: CBI.16 +rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + entity.addEngine(Engine.RULE); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); + insert(entity); + }); + end + +rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("et al.")) + then + entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) + .forEach(entity -> { + entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + entity.addEngine(Engine.RULE); + insert(entity); + dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); + }); + end + + +// Rule unit: CBI.17 +rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon" + when + $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) + then + entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.addEngine(Engine.RULE); + entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); + insert(entity); + }); + end + +rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon" + when + $section: Section(!hasTables(), containsString("Species:"), containsString("Source:")) + then + entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) + .forEach(entity -> { + entity.addEngine(Engine.RULE); + entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); + insert(entity); + }); + end + + +// Rule unit: CBI.20 +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); + laboratoryEntity.addEngine(Engine.RULE); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + laboratoryEntity.addEngine(Engine.RULE); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + + +//------------------------------------ PII rules ------------------------------------ + +// Rule unit: PII.0 +rule "PII.0.0: Redact all PII (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $pii: RedactionEntity(type == "PII", dictionaryEntry) + then + $pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "PII.0.1: Redact all PII (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $pii: RedactionEntity(type == "PII", dictionaryEntry) + then + $pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: PII.1 +rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> { + emailEntity.addEngine(Engine.RULE); + emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(emailEntity); + }); + end + +rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("@")) + then + entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) + .forEach(emailEntity -> { + emailEntity.addEngine(Engine.RULE); + emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(emailEntity); + }); + end + + +// Rule unit: PII.2 +rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> { + contactEntity.addEngine(Engine.RULE); + contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + insert(contactEntity); + }); + end + +rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Ph.") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) + .forEach(contactEntity -> { + contactEntity.addEngine(Engine.RULE); + contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + insert(contactEntity); + }); + end + + +// Rule unit: PII.9 +rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.addEngine(Engine.RULE); + insert(authorEntity); + }); + end + +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.addEngine(Engine.RULE); + insert(authorEntity); + }); + end + +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + authorEntity.addEngine(Engine.RULE); + insert(authorEntity); + }); + end + +rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:")) + then + entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) + .forEach(authorEntity -> { + authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + authorEntity.addEngine(Engine.RULE); + insert(authorEntity); + }); + end + + +//------------------------------------ Other rules ------------------------------------ + +// Rule unit: ETC.0 +rule "ETC.0.0: Purity Hint" + when + $section: Section(containsStringIgnoreCase("purity")) + then + entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) + .forEach(hint -> { + hint.addEngine(Engine.RULE); + hint.skip("ETC.0.0", ""); + }); + end + + +// Rule unit: ETC.2 +rule "ETC.2.0: Redact signatures (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "ETC.2.0: Redact signatures (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $signature: Image(imageType == ImageType.SIGNATURE) + then + $signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: ETC.3 +rule "ETC.3.0: Redact logos (vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $logo: Image(imageType == ImageType.LOGO) + then + $logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + end + +rule "ETC.3.1: Redact logos (non vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $logo: Image(imageType == ImageType.LOGO) + then + $logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + end + + +// Rule unit: ETC.5 +rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confidential'" + when + not FileAttribute(label == "Confidentiality", value == "confidential") + $dossierRedaction: RedactionEntity(type == "dossier_redaction") + then + $dossierRedaction.setIgnored(true); + update($dossierRedaction); + $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); + end + + +//------------------------------------ AI rules ------------------------------------ + +// Rule unit: AI.0 +rule "AI.0.0: add all NER Entities of type CBI_author" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("CBI_author")) + then + nerEntities.streamEntitiesOfType("CBI_author") + .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) + .forEach(entity -> insert(entity)); + end + + +// Rule unit: AI.1 +rule "AI.1.0: combine and add NER Entities as CBI_address" + salience 999 + when + nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) + then + nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) + .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) + .forEach(entity -> { + entity.addEngine(Engine.NER); + insert(entity); + }); + end + + +//------------------------------------ Manual redaction rules ------------------------------------ + +// Rule unit: MAN.0 +rule "MAN.0.0: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId) + $entityToBeResized: RedactionEntity(matchesAnnotationId($id)) + then + manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + + +// Rule unit: MAN.1 +rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.setIgnored(true); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) + $imageEntityToBeRemoved: Image($id == id) + then + $imageEntityToBeRemoved.setIgnored(true); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); + end + + +// Rule unit: MAN.2 +rule "MAN.2.0: Apply force redaction" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) + $entityToForce: RedactionEntity(matchesAnnotationId($id)) + then + $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); + update($entityToForce); + retract($force); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + end + + +// Rule unit: MAN.3 +rule "MAN.3.0: Apply image recategorization" + salience 128 + when + $recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); + update($imageToBeRecategorized); + retract($recategorization); + update($imageToBeRecategorized.getParent()); + end + + +//------------------------------------ Entity merging rules ------------------------------------ + +// Rule unit: X.0 +rule "X.0.0: remove Entity contained by Entity of same type" + salience 65 + when + $larger: RedactionEntity($type: type, $entityType: entityType) + $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger) + then + $contained.removeFromGraph(); + retract($contained); + end + + +// Rule unit: X.1 +rule "X.1.0: merge intersecting Entities of same type" + salience 64 + when + $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger) + $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger) + then + $first.removeFromGraph(); + $second.removeFromGraph(); + RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document); + retract($first); + retract($second); + insert(mergedEntity); + end + + +// Rule unit: X.2 +rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE) + $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.removeFromGraph(); + retract($entity) + end + + +// Rule unit: X.3 +rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION) + $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + + +// Rule unit: X.4 +rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.removeFromGraph(); + retract($recommendation); + end + + +// Rule unit: X.5 +rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: RedactionEntity(entityType == EntityType.ENTITY) + $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) + then + $recommendation.removeFromGraph(); + retract($recommendation); + end + + +// Rule unit: X.6 +rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY" + salience 32 + when + $higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY) + $lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger) + then + $lowerRank.removeFromGraph(); + retract($lowerRank); + end + + +//------------------------------------ File attributes rules ------------------------------------ + +// Rule unit: FA.1 +rule "FA.1.0: remove duplicate FileAttributes" + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end + + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LDS.0 +rule "LDS.0.0: run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) + .forEach(entity -> { + entity.addEngine(Engine.RULE); + insert(entity); + }); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index 2e62d4b1..c41c0240 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -1341,21 +1341,27 @@ rule "MAN.0.0: Apply manual resize redaction" rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id)) then $entityToBeRemoved.setIgnored(true); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" salience 128 when - IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) + $idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId) not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null) $imageEntityToBeRemoved: Image($id == id) then $imageEntityToBeRemoved.setIgnored(true); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); end