From d34921acd803038650f2e90bf1b2fecd995d3e15 Mon Sep 17 00:00:00 2001 From: maverickstuder Date: Thu, 28 Nov 2024 09:23:26 +0100 Subject: [PATCH] RED-10200: Spike: Performant update logic for facts in working memory --- .../server/model/document/DocumentTree.java | 2 +- .../entity/AbstractBidirectionalRelation.java | 16 ++ .../document/entity/AbstractRelation.java | 13 + .../entity/BidirectionalRelation.java | 7 + .../model/document/entity/Containment.java | 10 + .../model/document/entity/Equality.java | 17 ++ .../server/model/document/entity/IEntity.java | 10 +- .../document/entity/IKieSessionUpdater.java | 24 +- .../model/document/entity/Intersection.java | 17 ++ .../model/document/entity/Relation.java | 9 + .../model/document/entity/RelationStore.java | 79 ++++++ .../model/document/entity/TextEntity.java | 12 +- .../v1/server/model/document/nodes/Image.java | 6 + .../v1/server/model/PrecursorEntity.java | 8 +- .../service/DictionarySearchService.java | 4 +- .../server/service/EntityFindingUtility.java | 5 +- .../document/EntityCreationService.java | 34 ++- .../drools/EntityDroolsExecutionService.java | 17 +- .../service/drools/KieSessionUpdater.java | 34 ++- .../resources/drools/all_rules_documine.drl | 180 ++++++++---- .../v1/server/AnalysisEnd2EndTest.java | 3 +- ...cumentIEntityInsertionIntegrationTest.java | 5 +- .../DocumentPerformanceIntegrationTest.java | 5 +- .../graph/SearchImplementationTest.java | 8 +- .../v1/server/document/graph/TableTest.java | 5 +- .../ManualChangesEnd2EndTest.java | 7 +- .../manualchanges/ManualChangesUnitTest.java | 6 +- .../manualchanges/PrecursorEntityTest.java | 8 +- .../adapter/NerEntitiesAdapterTest.java | 5 +- .../v1/server/rules/RulesIntegrationTest.java | 5 +- .../resources/drools/acceptance_rules.drl | 236 ++++++++++------ .../drools/all_redact_manager_rules.drl | 252 +++++++++++------ .../test/resources/drools/documine_flora.drl | 180 ++++++++---- .../resources/drools/efsa_sanitisation.drl | 236 ++++++++++------ .../drools/manual_redaction_rules.drl | 234 ++++++++++------ .../src/test/resources/drools/rules.drl | 255 +++++++++++------ .../src/test/resources/drools/rules_v2.drl | 234 ++++++++++------ .../src/test/resources/drools/table_demo.drl | 234 ++++++++++------ .../src/test/resources/drools/test_rules.drl | 263 ++++++++++++------ .../src/test/resources/files/syngenta | 2 +- .../EFSA_sanitisation_GFL_v1/rules.drl | 234 ++++++++++------ .../resources/all_redact_manager_rules.drl | 252 +++++++++++------ .../src/main/resources/all_rules_documine.drl | 180 ++++++++---- 43 files changed, 2308 insertions(+), 1045 deletions(-) create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java create mode 100644 redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java index 5c7ba341..3e990759 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/DocumentTree.java @@ -35,7 +35,7 @@ public class DocumentTree { public DocumentTree(Document document) { - root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build(); + this.root = Entry.builder().treeId(Collections.emptyList()).children(new LinkedList<>()).node(document).build(); } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java new file mode 100644 index 00000000..55aae869 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractBidirectionalRelation.java @@ -0,0 +1,16 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public abstract class AbstractBidirectionalRelation implements BidirectionalRelation { + + protected final TextEntity a; + protected final TextEntity b; + + @Override + public abstract BidirectionalRelation invert(); + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java new file mode 100644 index 00000000..cee877ca --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/AbstractRelation.java @@ -0,0 +1,13 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public abstract class AbstractRelation implements Relation { + + protected final TextEntity a; + protected final TextEntity b; + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java new file mode 100644 index 00000000..57b70509 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/BidirectionalRelation.java @@ -0,0 +1,7 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public interface BidirectionalRelation extends Relation { + + BidirectionalRelation invert(); + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java new file mode 100644 index 00000000..2d621959 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Containment.java @@ -0,0 +1,10 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Containment extends AbstractRelation { + + public Containment(TextEntity container, TextEntity contained) { + + super(container, contained); + } + +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java new file mode 100644 index 00000000..5852ed50 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Equality.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Equality extends AbstractBidirectionalRelation { + + public Equality(TextEntity a, TextEntity b) { + + super(a, b); + } + + + @Override + public Equality invert() { + + return new Equality(b, a); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java index 16c27f2a..65c9dea2 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IEntity.java @@ -56,6 +56,10 @@ public interface IEntity { * Marks this entity and all its intersecting nodes as updated */ void update(); + /** + * Marks this entity and all its intersecting nodes as removed + */ + void remove(); /** * An Entity is valid, when it active and not a false recommendation, a false positive or a dictionary removal. @@ -350,7 +354,11 @@ public interface IEntity { if (valid() == valid) { return; } - update(); + if(!removed()) { + update(); + } else { + remove(); + } } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IKieSessionUpdater.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IKieSessionUpdater.java index 3e4b4bda..9089e2fb 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IKieSessionUpdater.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/IKieSessionUpdater.java @@ -4,24 +4,18 @@ import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; public interface IKieSessionUpdater { - /** - * Inserts a TextEntity into the KieSession and updates intersecting nodes. - * - * @param textEntity the TextEntity to insert - */ void insert(TextEntity textEntity); - /** - * Updates a TextEntity in the KieSession and updates intersecting nodes. - * - * @param textEntity the TextEntity to update - */ + void update(TextEntity textEntity); - /** - * Updates an Image in the KieSession and recursively updates its parent nodes. - * - * @param image the Image to update - */ + void update(Image image); + + + void remove(TextEntity textEntity); + + + void remove(Image image); + } diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java new file mode 100644 index 00000000..00e57fe5 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Intersection.java @@ -0,0 +1,17 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public class Intersection extends AbstractBidirectionalRelation { + + public Intersection(TextEntity a, TextEntity b) { + + super(a, b); + } + + + @Override + public Intersection invert() { + + return new Intersection(b, a); + } + +} \ No newline at end of file diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java new file mode 100644 index 00000000..cfbe1662 --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/Relation.java @@ -0,0 +1,9 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +public interface Relation { + + TextEntity getA(); + + + TextEntity getB(); +} diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java new file mode 100644 index 00000000..296d632e --- /dev/null +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/RelationStore.java @@ -0,0 +1,79 @@ +package com.iqser.red.service.redaction.v1.server.model.document.entity; + +import java.util.*; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.stereotype.Service; + +@Service +public class RelationStore { + + // +// private final Map, Relation> relations = new HashMap<>(); + private final Map> entityRelations = new HashMap<>(); + + + public void addRelation(Relation relation) { + + String idA = relation.getA().getId(); + String idB = relation.getB().getId(); + + Pair key = Pair.of(idA, idB); +// relations.put(key, relation); + + if (relation instanceof BidirectionalRelation bidirectionalRelation) { + +// BidirectionalRelation invertedRelation = bidirectionalRelation.invert(); +// Pair invertedKey = Pair.of(idB, idA); +// relations.put(invertedKey, invertedRelation); + entityRelations.computeIfAbsent(idB, k -> new HashSet<>()).add(relation); + } + + entityRelations.computeIfAbsent(idA, k -> new HashSet<>()).add(relation); + } + + + public void removeRelation(Relation relation) { + + String idA = relation.getA().getId(); + String idB = relation.getB().getId(); + + Pair key = Pair.of(idA, idB); +// relations.remove(key); + + if (relation instanceof BidirectionalRelation) { + +// Pair invertedKey = Pair.of(idB, idA); +// relations.remove(invertedKey); + entityRelations.getOrDefault(idB, Collections.emptySet()).remove(relation); + } + + entityRelations.getOrDefault(idA, Collections.emptySet()).remove(relation); + } + + + public Set getRelations(String entityId) { + + return entityRelations.getOrDefault(entityId, Collections.emptySet()); + } + +// +// public Relation getRelation(String idA, String idB) { +// +// return relations.get(Pair.of(idA, idB)); +// } + + + public Set getAllRelations(Set entityIds) { + + return entityRelations.entrySet() + .stream() + .filter(entry -> entityIds.contains(entry.getKey())) + .flatMap(entry -> entry.getValue() + .stream()) + .collect(Collectors.toSet()); + } + +} + diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java index d196b335..dbdbbf46 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/TextEntity.java @@ -52,7 +52,8 @@ public class TextEntity implements IEntity { @Builder.Default final PriorityQueue matchedRuleList = new PriorityQueue<>(); - final ManualChangeOverwrite manualOverwrite; + @Builder.Default + final ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite(); boolean dictionaryEntry; boolean dossierDictionaryEntry; @@ -71,7 +72,8 @@ public class TextEntity implements IEntity { List intersectingNodes = new LinkedList<>(); SemanticNode deepestFullyContainingNode; - List relations = new LinkedList<>(); +// @Builder.Default +// Set relations = new HashSet<>(); public static TextEntity initialEntityNode(TextRange textRange, String type, EntityType entityType, SemanticNode node) { @@ -166,6 +168,7 @@ public class TextEntity implements IEntity { intersectingNodes.forEach(node -> node.getEntities().remove(this)); pages.forEach(page -> page.getEntities().remove(this)); intersectingNodes = new LinkedList<>(); + //relations = new HashSet<>(); deepestFullyContainingNode = null; pages = new HashSet<>(); remove("FINAL.0.0", "removed completely"); @@ -322,6 +325,11 @@ public class TextEntity implements IEntity { getKieSessionUpdater().ifPresent(updater -> updater.update(this)); } + public void remove() { + + getKieSessionUpdater().ifPresent(updater -> updater.remove(this)); + } + private @NonNull Optional getKieSessionUpdater() { diff --git a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index fed4be8d..a75fee4b 100644 --- a/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/document/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -117,6 +117,12 @@ public class Image extends AbstractSemanticNode implements IEntity { getKieSessionUpdater().ifPresent(updater -> updater.update(this)); } + @Override + public void remove() { + + getKieSessionUpdater().ifPresent(updater -> updater.remove(this)); + } + @Override public String toString() { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java index ac1c0103..82b0c8da 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/PrecursorEntity.java @@ -52,7 +52,8 @@ public class PrecursorEntity implements IEntity { @Builder.Default PriorityQueue matchedRuleList = new PriorityQueue<>(); - ManualChangeOverwrite manualOverwrite; + @Builder.Default + ManualChangeOverwrite manualOverwrite = new ManualChangeOverwrite(); public static PrecursorEntity fromManualRedactionEntry(ManualRedactionEntry manualRedactionEntry, boolean hint) { @@ -187,6 +188,11 @@ public class PrecursorEntity implements IEntity { // not in KieSession, do nothing } + @Override + public void remove() { + + // not in KieSession, do nothing + } /** diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java index 478c4a6f..4cd49974 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/DictionarySearchService.java @@ -8,6 +8,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityEnrichmentService; @@ -25,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; public class DictionarySearchService { EntityEnrichmentService entityEnrichmentService; + RelationStore relationStore; @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") @@ -37,7 +39,7 @@ public class DictionarySearchService { @Observed(name = "DictionarySearchService", contextualName = "add-dictionary-entries") public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); dictionary.getDictionarySearch().getBoundaries(node.getTextBlock()) .filter(boundary -> entityCreationService.isValidEntityTextRange(node.getTextBlock(), boundary.textRange())) .forEach(match -> { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java index 5e760c87..c06c4838 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityFindingUtility.java @@ -24,6 +24,7 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; import com.iqser.red.service.redaction.v1.server.model.document.entity.RectangleWithPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -43,9 +44,9 @@ public class EntityFindingUtility { @Autowired - public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService) { + public EntityFindingUtility(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java index 7519e221..4b19b877 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/EntityCreationService.java @@ -29,10 +29,15 @@ import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplemen import com.iqser.red.service.redaction.v1.server.model.document.ConsecutiveBoundaryCollector; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality; import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection; import com.iqser.red.service.redaction.v1.server.model.document.entity.ManualChangeOverwrite; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -51,12 +56,14 @@ import lombok.extern.slf4j.Slf4j; public class EntityCreationService { private final EntityEnrichmentService entityEnrichmentService; + private final RelationStore relationStore; private final Set nodesInKieSession; // empty set means all nodes are in kieSession - public EntityCreationService(EntityEnrichmentService entityEnrichmentService) { + public EntityCreationService(EntityEnrichmentService entityEnrichmentService, RelationStore relationStore) { this.entityEnrichmentService = entityEnrichmentService; + this.relationStore = relationStore; this.nodesInKieSession = Collections.emptySet(); } @@ -1517,7 +1524,30 @@ public class EntityCreationService { addToPages(entity); addEntityToNodeEntitySets(entity); - // compute relationships, by looping through deepestFullyContainingNode's entities + + if(relationStore != null) { + //List relations = new LinkedList<>(); + // compute relationships, by looping through deepestFullyContainingNode's entities + for (TextEntity textEntity : entity.getDeepestFullyContainingNode().getEntities()) { + if (entity.intersects(textEntity) && !entity.equals(textEntity)) { + Relation relation; + if (textEntity.getTextRange().equals(entity.getTextRange())) { + relation = new Equality(entity, textEntity); + } else if (textEntity.containedBy(entity)) { + relation = new Containment(textEntity, entity); + } else if (entity.containedBy(textEntity)) { + relation = new Containment(entity, textEntity); + } else { + relation = new Intersection(entity, textEntity); + } + relationStore.addRelation(relation); + //relations.add(relation); + //textEntity.getRelations().add(relation); + } + } + //entity.getRelations().addAll(relations); + } + return true; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java index cfce5fa4..127d523d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/EntityDroolsExecutionService.java @@ -2,6 +2,7 @@ package com.iqser.red.service.redaction.v1.server.service.drools; import static com.iqser.red.service.redaction.v1.server.service.drools.ComponentDroolsExecutionService.RULES_LOGGER_GLOBAL; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -10,6 +11,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieSession; @@ -28,6 +30,8 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger; import com.iqser.red.service.redaction.v1.server.logger.TrackingAgendaEventListener; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; @@ -51,6 +55,7 @@ import lombok.extern.slf4j.Slf4j; public class EntityDroolsExecutionService { EntityEnrichmentService entityEnrichmentService; + RelationStore relationStore; ObservationRegistry observationRegistry; ManualChangesApplicationService manualChangesApplicationService; RedactionServiceSettings settings; @@ -94,12 +99,13 @@ public class EntityDroolsExecutionService { KieSession kieSession = kieContainer.newKieSession(); - Set nodesInKieSession = new HashSet<>(); + Set nodesInKieSession = sectionsToAnalyze.size() == document.streamAllSubNodes() + .count() ? new HashSet<>() : buildSet(sectionsToAnalyze, document); - KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(nodesInKieSession, kieSession); + KieSessionUpdater kieSessionUpdater = new KieSessionUpdater(kieSession, relationStore, nodesInKieSession); document.setKieSessionUpdater(kieSessionUpdater); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, nodesInKieSession); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore, nodesInKieSession); RulesLogger logger = new RulesLogger(webSocketService, context); if (settings.isDroolsDebug()) { logger.enableAgendaTracking(); @@ -127,6 +133,11 @@ public class EntityDroolsExecutionService { document.getEntities() .forEach(kieSession::insert); + relationStore.getAllRelations(document.getEntities() + .stream() + .map(TextEntity::getId) + .collect(Collectors.toSet())) + .forEach(kieSession::insert); System.out.println("after getEntities insert : " + kieSession.getFactCount()); document.getPages() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java index 025c9cc9..0503bcd6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/KieSessionUpdater.java @@ -6,6 +6,8 @@ import org.kie.api.runtime.KieSession; import org.kie.api.runtime.rule.FactHandle; import com.iqser.red.service.redaction.v1.server.model.document.entity.IKieSessionUpdater; +import com.iqser.red.service.redaction.v1.server.model.document.entity.Relation; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; @@ -18,14 +20,17 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE) public class KieSessionUpdater implements IKieSessionUpdater { - Set nodesInKieSession; KieSession kieSession; + RelationStore relationStore; + Set nodesInKieSession; public void insert(TextEntity textEntity) { kieSession.insert(textEntity); updateIntersectingNodes(textEntity); + relationStore.getRelations(textEntity.getId()) + .forEach(kieSession::insert); } @@ -33,6 +38,8 @@ public class KieSessionUpdater implements IKieSessionUpdater { kieSession.update(kieSession.getFactHandle(textEntity), textEntity); updateIntersectingNodes(textEntity); + relationStore.getRelations(textEntity.getId()) + .forEach(r -> kieSession.update(kieSession.getFactHandle(r), r)); } @@ -47,6 +54,31 @@ public class KieSessionUpdater implements IKieSessionUpdater { } + public void remove(TextEntity textEntity) { + + kieSession.delete(kieSession.getFactHandle(textEntity)); + updateIntersectingNodes(textEntity); + relationStore.getRelations(textEntity.getId()) + .forEach(r -> { + FactHandle factHandle = kieSession.getFactHandle(r); + if (factHandle != null) { + kieSession.delete(factHandle); + } + }); + } + + + public void remove(Image image) { + + kieSession.delete(kieSession.getFactHandle(image)); + SemanticNode parent = image; + while (parent.hasParent()) { + parent = parent.getParent(); + kieSession.update(kieSession.getFactHandle(parent), parent); + } + } + + private void updateIntersectingNodes(TextEntity textEntity) { textEntity.getIntersectingNodes() diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl index b1c04d70..7e26fafb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl @@ -1311,8 +1311,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1337,9 +1335,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1363,8 +1359,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1389,9 +1383,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -1447,7 +1439,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -1458,23 +1449,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -1482,34 +1494,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -1517,12 +1545,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1530,21 +1564,32 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end @@ -1563,22 +1608,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1607,11 +1663,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1624,7 +1685,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java index 84ed4c3b..721bc2c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisEnd2EndTest.java @@ -34,6 +34,7 @@ import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.amqp.rabbit.core.RabbitTemplate; @@ -93,7 +94,7 @@ import lombok.extern.slf4j.Slf4j; @ExtendWith(SpringExtension.class) @SpringBootTest(classes = Application.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @Import(AbstractRedactionIntegrationTest.TestConfiguration.class) -//@Disabled +@Disabled /* * This test is meant to be used directly with a download from blob storage (e.g. minio). You need to define the dossier template you want to use by supplying an absolute path. * The dossier template will then be parsed for dictionaries, colors, entities, and rules. This is defined for the all tests once. diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java index 5ce26613..aba510f7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentIEntityInsertionIntegrationTest.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.document.DocumentTree; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; @@ -36,6 +37,8 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; private EntityCreationService entityCreationService; @Mock @@ -46,7 +49,7 @@ public class DocumentIEntityInsertionIntegrationTest extends BuildDocumentIntegr public void createEntityCreationService() { MockitoAnnotations.initMocks(this); - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java index a0de089d..3e6fb4a6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java @@ -30,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.logger.Context; import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionarySearch; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; @@ -55,6 +56,8 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; private EntityCreationService entityCreationService; @Autowired @@ -68,7 +71,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest { @BeforeEach public void stubClients() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); TenantContext.setTenantId("redaction"); when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(System.currentTimeMillis()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java index a98865ae..8abb56d3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/SearchImplementationTest.java @@ -9,6 +9,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.redaction.v1.server.model.dictionary.SearchImplementation; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; @@ -19,6 +20,9 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; + @Test public void testSearchImplementationWithPunctuation() { @@ -26,7 +30,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); SearchImplementation searchImplementation = new SearchImplementation(List.of("Kuhn, J. O."), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); List entities = entityCreationService.bySearchImplementation(searchImplementation, "CBI_author", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); @@ -38,7 +42,7 @@ public class SearchImplementationTest extends BuildDocumentIntegrationTest { Document document = buildGraph("files/syngenta/CustomerFiles/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections"); SearchImplementation searchImplementation = new SearchImplementation(List.of("mydossierredaction"), true); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); List entities = entityCreationService.bySearchImplementation(searchImplementation, "dossier_redaction", EntityType.ENTITY, document) .toList(); assertEquals(2, entities.size()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java index e2e33390..1c89feb3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/TableTest.java @@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired; import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; @@ -35,6 +36,8 @@ public class TableTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; private EntityCreationService entityCreationService; @@ -52,7 +55,7 @@ public class TableTest extends BuildDocumentIntegrationTest { @BeforeEach public void createTable() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); String fileName = "files/Minimal Examples/BasicTable.pdf"; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java index 6ea85b4c..0b486730 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java @@ -51,6 +51,7 @@ import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest; import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse; import com.iqser.red.service.redaction.v1.server.mapper.DocumentGraphMapper; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; @@ -67,16 +68,20 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest { private static final String RULES = loadFromClassPath("drools/acceptance_rules.drl"); private static final String DM_RULES = loadFromClassPath("drools/documine_flora.drl"); + @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; + private EntityCreationService entityCreationService; @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java index b370bf1e..07ba02bd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesUnitTest.java @@ -17,6 +17,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; @@ -27,13 +28,16 @@ public class ManualChangesUnitTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; + private EntityCreationService entityCreationService; @BeforeEach public void createServices() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java index 212f5ae9..32c1e701 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java @@ -30,6 +30,7 @@ import com.iqser.red.service.redaction.v1.server.model.PrecursorEntity; import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryVersion; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.IEntity; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.DictionaryService; @@ -45,6 +46,9 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; + @Autowired private EntityFromPrecursorCreationService entityFromPrecursorCreationService; @@ -104,7 +108,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { public void createFoundManualRedaction2() { Document document = buildGraph("files/Minimal Examples/TestPunctuation"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); List tempEntities = entityCreationService.byString("Kuhn, J. O.", "CBI_author", EntityType.ENTITY, document) .toList(); @@ -159,7 +163,7 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest { private DocumentAndEntity createFoundManualRedaction() { Document document = buildGraph("files/syngenta/CustomerFiles/VV-919901.pdf"); - EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService); + EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); List tempEntities = entityCreationService.byString("To: Syngenta Ltd.", "temp", EntityType.ENTITY, document) .toList(); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java index 3c919839..8ff6d22a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java @@ -29,6 +29,7 @@ import com.iqser.red.service.redaction.v1.server.model.NerEntities; import com.iqser.red.service.redaction.v1.server.model.document.TextRange; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.PositionOnPage; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; @@ -44,6 +45,8 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { @Autowired private EntityEnrichmentService entityEnrichmentService; + @Autowired + private RelationStore relationStore; private EntityCreationService entityCreationService; @@ -58,7 +61,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest { @BeforeEach public void createEntityCreationService() { - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java index 5117b975..7276acb2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/rules/RulesIntegrationTest.java @@ -22,6 +22,7 @@ import org.springframework.context.annotation.Import; import com.iqser.red.service.redaction.v1.server.AbstractRedactionIntegrationTest; import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest; import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.document.entity.RelationStore; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; @@ -36,6 +37,8 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { @Autowired protected EntityEnrichmentService entityEnrichmentService; @Autowired + protected RelationStore relationStore; + @Autowired protected ManualChangesApplicationService manualChangesApplicationService; protected EntityCreationService entityCreationService; protected KieSession kieSession; @@ -82,7 +85,7 @@ public class RulesIntegrationTest extends BuildDocumentIntegrationTest { Dictionary dict = Mockito.mock(Dictionary.class); kieSession = kieContainer.newKieSession(); - entityCreationService = new EntityCreationService(entityEnrichmentService); + entityCreationService = new EntityCreationService(entityEnrichmentService, relationStore); kieSession.setGlobal("manualChangesApplicationService", manualChangesApplicationService); kieSession.setGlobal("entityCreationService", entityCreationService); kieSession.setGlobal("dictionary", dict); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 328a01fb..d5535a2a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -985,7 +985,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -995,7 +994,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1114,8 +1112,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1140,9 +1136,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1166,8 +1160,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1192,9 +1184,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -1250,7 +1240,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -1261,23 +1250,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -1285,34 +1295,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -1320,12 +1346,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1333,45 +1365,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -1379,22 +1432,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1420,25 +1484,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1451,7 +1526,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 06aeeb78..81897adc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -537,7 +537,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations" $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); - retract($entity) end rule "CBI.13.1: Redacted because Section contains a vertebrate" @@ -765,7 +764,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -779,7 +777,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -1613,7 +1610,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -1623,7 +1619,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1881,8 +1876,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1907,9 +1900,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1933,8 +1924,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1959,9 +1948,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -2017,7 +2004,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -2028,23 +2014,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -2052,34 +2059,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -2087,12 +2110,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -2100,45 +2129,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -2146,22 +2196,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -2190,32 +2251,48 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -2228,7 +2305,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 8b3442c1..29ae8032 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1248,8 +1248,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1274,9 +1272,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1300,8 +1296,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1326,9 +1320,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -1384,7 +1376,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -1395,23 +1386,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -1419,34 +1431,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -1454,12 +1482,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1467,21 +1501,32 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end @@ -1500,22 +1545,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1544,11 +1600,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1561,7 +1622,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 796d0a07..cacfe1e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -694,7 +694,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -704,7 +703,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -841,8 +839,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -867,9 +863,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -893,8 +887,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -919,9 +911,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -977,7 +967,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -988,23 +977,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -1012,34 +1022,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -1047,12 +1073,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1060,45 +1092,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -1106,22 +1159,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1147,25 +1211,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1178,7 +1253,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 47594a31..09157bdc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -160,8 +160,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -186,9 +184,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -212,8 +208,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -238,9 +232,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -296,7 +288,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -307,23 +298,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -331,34 +343,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -366,12 +394,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -379,45 +413,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -425,22 +480,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -466,25 +532,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -497,7 +574,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 28159a70..57af3cd3 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -19,6 +19,9 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.*; import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.Containment +import com.iqser.red.service.redaction.v1.server.model.document.entity.Equality +import com.iqser.red.service.redaction.v1.server.model.document.entity.Intersection import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; @@ -447,7 +450,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -461,7 +463,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -973,7 +974,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1135,7 +1135,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); end rule "MAN.0.1: Apply manual resize redaction" @@ -1160,7 +1159,6 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); end @@ -1185,7 +1183,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); retract($force); end @@ -1211,7 +1208,6 @@ rule "MAN.3.0: Apply entity recategorization" $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -1267,150 +1263,221 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end //------------------------------------ Entity merging rules ------------------------------------ - // Rule unit: X.0 rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end - // Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end - // Rule unit: X.4 rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end - // Rule unit: X.5 rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end - // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end - // Rule unit: X.8 rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end - // Rule unit: X.9 rule "X.9.0: Merge mostly contained signatures" when @@ -1431,30 +1498,56 @@ rule "X.10.0: remove false positives of ai" $aiSignature.remove("X.10.0", "Removed because false positive"); end - // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" + salience 64 + when + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) + then + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); + end + +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end + //------------------------------------ Dictionary merging rules ------------------------------------ // Rule unit: DICT.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 17e1c255..7ebdfa5a 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -231,8 +231,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -257,9 +255,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -283,8 +279,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -309,9 +303,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -354,7 +346,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -365,23 +356,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -389,34 +401,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -424,12 +452,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -437,45 +471,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -483,22 +538,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -524,25 +590,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -555,7 +632,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 0166e897..a4cce953 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -310,8 +310,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -336,9 +334,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -362,8 +358,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -388,9 +382,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -446,7 +438,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -457,23 +448,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -481,34 +493,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -516,12 +544,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -529,45 +563,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -586,22 +641,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -627,25 +693,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -658,7 +735,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 9c5c810d..dda0ce3f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -210,8 +210,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -236,9 +234,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -262,8 +258,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -288,9 +282,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -346,7 +338,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -357,23 +348,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -381,55 +393,82 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -437,12 +476,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -450,45 +495,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -507,22 +573,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -548,25 +625,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -579,7 +667,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta index 5705cc07..57e6e0dd 160000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/syngenta @@ -1 +1 @@ -Subproject commit 5705cc0782605fdca5dfff134b436f7143c9e421 +Subproject commit 57e6e0dd3c08a3a65ec59b5dfb70f0f77ebcc7c7 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index a8d7470b..e9fc33c6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -723,8 +723,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -749,9 +747,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -775,8 +771,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -801,9 +795,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -859,7 +851,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -870,23 +861,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -894,34 +906,50 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -929,12 +957,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -942,45 +976,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -988,22 +1043,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -1029,25 +1095,36 @@ rule "X.10.0: remove false positives of ai" // Rule unit: X.11 -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -1060,7 +1137,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 1fc73ad7..f7954135 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -541,7 +541,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations" $entity: TextEntity(type() == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); - retract($entity) end // from CBI.3.0 @@ -775,7 +774,6 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials"); - retract($entityToExpand); }); end @@ -789,7 +787,6 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); $entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix"); - retract($entityToExpand); }); end @@ -1631,7 +1628,6 @@ rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidentia $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential"); - $dossierRedaction.getIntersectingNodes().forEach(node -> update(node)); end rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'" @@ -1641,7 +1637,6 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi $dossierRedaction: TextEntity(type() == "dossier_redaction") then $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential"); - retract($dossierRedaction); end @@ -1899,8 +1894,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1925,9 +1918,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1951,8 +1942,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1977,9 +1966,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -2036,7 +2023,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -2047,23 +2033,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end rule "X.0.4: Remove Entity contained by Entity of same type" @@ -2080,35 +2087,51 @@ rule "X.0.4: Remove Entity contained by Entity of same type" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -2116,12 +2139,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -2129,45 +2158,66 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end // Rule unit: X.6 -rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT" +rule "X.6.0: Remove Entity of lower rank when contained by entity of type ENTITY or HINT" salience 32 when - $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval(dictionary.getDictionaryRank($b.type()) < dictionary.getDictionaryRank($a.type())), + !$b.hasManualChanges() + ) then - $lowerRank.getIntersectingNodes().forEach(node -> update(node)); - $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT"); - retract($lowerRank); + $b.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT"); end -rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range" +rule "X.6.1: Remove Entity when contained in another entity of type ENTITY or HINT with larger text range" salience 32 when - $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.type() != $a.type(), + eval($a.getTextRange().length() > $b.getTextRange().length()), + !$b.hasManualChanges() + ) then - $inner.getIntersectingNodes().forEach(node -> update(node)); - $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"); - retract($inner); + $b.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range"); end @@ -2175,22 +2225,33 @@ rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or H rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end @@ -2219,34 +2280,50 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end -rule "X.11.1: Remove non manual entity which intersects with a manual entity" +rule "X.11.1: Remove non-manual entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.engines not contains Engine.MANUAL + ) then - $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity"); - retract($nonManualEntity); + $b.remove("X.11.1", "remove entity which intersects with a manual entity"); end -rule "X.11.2: Remove non manual entity which are equal to manual entity" +rule "X.11.2: Remove non-manual entity which is equal to manual entity" salience 70 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type()) - $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.MANUAL, + $a.active(), + $b.entityType == EntityType.ENTITY, + !$b.hasManualChanges(), + $b.engines not contains Engine.MANUAL + ) then - $manualEntity.addEngines($nonManualEntity.getEngines()); - $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity"); - retract($nonManualEntity); + $a.addEngines($b.getEngines()); + $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity"); end @@ -2259,7 +2336,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 3eac9dd4..bb8475de 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1315,8 +1315,6 @@ rule "MAN.0.0: Apply manual resize redaction" then manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction); retract($resizeRedaction); - update($entityToBeResized); - $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.0.1: Apply manual resize redaction" @@ -1341,9 +1339,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) then $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); - update($entityToBeRemoved); retract($idRemoval); - $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); end rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" @@ -1367,8 +1363,6 @@ rule "MAN.2.0: Apply force redaction" $entityToForce: TextEntity(matchesAnnotationId($id)) then $entityToForce.getManualOverwrite().addChange($force); - update($entityToForce); - $entityToForce.getIntersectingNodes().forEach(node -> update(node)); retract($force); end @@ -1393,9 +1387,7 @@ rule "MAN.3.0: Apply entity recategorization" not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type) then - $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); - update($entityToBeRecategorized); retract($recategorization); end @@ -1450,7 +1442,6 @@ rule "MAN.4.1: Apply legal basis change" $entityToBeChanged: TextEntity(matchesAnnotationId($id)) then $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); - update($entityToBeChanged) retract($legalBasisChange) end @@ -1461,23 +1452,44 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges()) - not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + !$a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) + not TextEntity( + getTextRange().equals($a.getTextRange()), + type() == $a.type(), + entityType == EntityType.DICTIONARY_REMOVAL, + engines contains Engine.DOSSIER_DICTIONARY, + !hasManualChanges() + ) then - $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); - retract($contained); + $b.remove("X.0.0", "remove Entity contained by Entity of same type"); end rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges()) - $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.entityType == $b.entityType, + $a != $b, + !$a.removed(), + $a.hasManualChanges(), + !$b.hasManualChanges(), + !$b.removed() + ) then - $contained.getIntersectingNodes().forEach(node -> update(node)); - $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); - retract($contained); + $b.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes"); end @@ -1485,35 +1497,51 @@ rule "X.0.1: Remove Entity contained by Entity of same type with manual changes" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.ENTITY, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); end rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" salience 64 when - $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active()) - $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_POSITIVE, + $a.active(), + $b.entityType == EntityType.HINT, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $entity.getIntersectingNodes().forEach(node -> update(node)); - $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); - retract($entity) + $b.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE"); end -// Rule unit: X.3 rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" salience 64 when - $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.FALSE_RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $b.type() == $a.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); - retract($recommendation); + $b.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); end @@ -1521,12 +1549,18 @@ rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type" salience 256 when - $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $entity.addEngines($recommendation.getEngines()); - $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); - retract($recommendation); + $a.addEngines($b.getEngines()); + $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"); end @@ -1534,11 +1568,16 @@ rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" salience 256 when - $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) - $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $intersection: Intersection( + $a: a, + $b: b, + ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT), + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); - retract($recommendation); + $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY"); end @@ -1546,11 +1585,17 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" salience 256 when - $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active()) - $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges()) + $containment: Containment( + $a: a, + $b: b, + $a.entityType == EntityType.RECOMMENDATION, + $a.active(), + $b.entityType == EntityType.RECOMMENDATION, + $a.type() != $b.type(), + !$b.hasManualChanges() + ) then - $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); - retract($recommendation); + $b.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"); end @@ -1569,22 +1614,33 @@ rule "X.7.0: Remove all images" rule "X.8.0: Remove Entity when text range and type equals to imported Entity" salience 257 when - $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active()) - $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED) + $equality: Equality( + $a: a, + $b: b, + $a.type() == $b.type(), + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); - $entity.addEngines($other.getEngines()); - retract($other); + $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity"); + $a.addEngines($b.getEngines()); end rule "X.8.1: Remove Entity when intersected by imported Entity" salience 256 when - $entity: TextEntity(engines contains Engine.IMPORTED, active()) - $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.IMPORTED, + $a.active(), + $b.engines not contains Engine.IMPORTED, + $a != $b + ) then - $other.remove("X.8.1", "remove Entity when intersected by imported Entity"); - retract($other); + $b.remove("X.8.1", "remove Entity when intersected by imported Entity"); end // Rule unit: X.9 @@ -1611,11 +1667,16 @@ rule "X.10.0: remove false positives of ai" rule "X.11.0: Remove dictionary entity which intersects with a manual entity" salience 64 when - $manualEntity: TextEntity(engines contains Engine.MANUAL, active()) - $dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL) + $intersection: Intersection( + $a: a, + $b: b, + $a.engines contains Engine.MANUAL, + $a.active(), + $b.dictionaryEntry, + $b.engines not contains Engine.MANUAL + ) then - $dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); - retract($dictionaryEntity); + $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity"); end @@ -1628,7 +1689,6 @@ rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dict $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY) $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges()) then - $entity.getIntersectingNodes().forEach(node -> update(node)); $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"); $entity.addEngine(Engine.DOSSIER_DICTIONARY); end