From 0bfd92846fa0052468d108b23f4edb1029c32334 Mon Sep 17 00:00:00 2001 From: Yannik Hampe Date: Wed, 28 Feb 2024 09:04:55 +0100 Subject: [PATCH] RED-8481: use visual layout parser for signature extraction --- .../v1/server/model/document/nodes/Image.java | 30 +++++++++++++++---- .../resources/drools/acceptance_rules.drl | 23 ++++++++++++++ .../drools/all_redact_manager_rules.drl | 23 ++++++++++++++ .../test/resources/drools/documine_flora.drl | 23 ++++++++++++++ .../drools/manual_redaction_rules.drl | 23 ++++++++++++++ .../src/test/resources/drools/rules.drl | 23 ++++++++++++++ .../src/test/resources/drools/rules_v2.drl | 23 ++++++++++++++ .../src/test/resources/drools/table_demo.drl | 23 ++++++++++++++ .../src/test/resources/drools/test_rules.drl | 23 ++++++++++++++ .../migration/RuleFileMigrator.java | 6 +++- .../resources/all_redact_manager_rules.drl | 20 +++++++++++++ .../src/main/resources/all_rules_documine.drl | 23 +++++++++++++- 12 files changed, 255 insertions(+), 8 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java index 102c9f20..59e024d9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/nodes/Image.java @@ -35,6 +35,7 @@ import lombok.experimental.FieldDefaults; @FieldDefaults(level = AccessLevel.PRIVATE) @EqualsAndHashCode(onlyExplicitlyIncluded = true) public class Image implements GenericSemanticNode, IEntity { + @Builder.Default Set engines = new HashSet<>(Set.of(LayoutEngine.ALGORITHM)); @@ -73,7 +74,9 @@ public class Image implements GenericSemanticNode, IEntity { @Override public TextBlock getTextBlock() { - return streamAllSubNodes().filter(SemanticNode::isLeaf).map(SemanticNode::getLeafTextBlock).collect(new TextBlockCollector()); + return streamAllSubNodes().filter(SemanticNode::isLeaf) + .map(SemanticNode::getLeafTextBlock) + .collect(new TextBlockCollector()); } @@ -94,7 +97,8 @@ public class Image implements GenericSemanticNode, IEntity { @Override public String type() { - return getManualOverwrite().getType().orElse(imageType.toString()); + return getManualOverwrite().getType() + .orElse(imageType.toString()); } @@ -126,15 +130,29 @@ public class Image implements GenericSemanticNode, IEntity { return name.charAt(0) + name.substring(1).toLowerCase(Locale.ENGLISH); } - public boolean mostlyContainedBy(Image image) { - Map bboxImage = image.getBBox(); - Map bbox = this.getBBox(); + + public boolean mostlyContainedBy(Image image, double containmentThreshold) { + + Map bboxImage = image.getBBox(); + Map bbox = this.getBBox(); Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page)); double calculatedIntersection = intersection.getWidth() * intersection.getHeight(); double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight(); - return (calculatedIntersection / area) > 0.8; + return (calculatedIntersection / area) > containmentThreshold; } + + public boolean mostlyContains(Image image, double containmentThreshold) { + + Map bboxImage = image.getBBox(); + Map bbox = this.getBBox(); + Rectangle2D intersection = bboxImage.get(this.page).createIntersection(bbox.get(this.page)); + double calculatedIntersection = intersection.getWidth() * intersection.getHeight(); + double area = bbox.get(this.page).getWidth() * bbox.get(this.page).getHeight(); + return (area / calculatedIntersection) > containmentThreshold; + } + + public int length() { return 0; diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index ff854a43..3d278a3b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1013,6 +1015,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index b1fbf077..5e3663e2 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1576,6 +1578,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 228989b6..54f8b265 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1414,6 +1416,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl index 3cb50691..7a018c3c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -255,6 +257,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ Local dictionary search rules ------------------------------------ // Rule unit: LDS.0 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index aed445b2..dd53a53e 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1166,6 +1168,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 2984605c..3347cc6f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -379,6 +381,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 61cf03cd..029e649c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -500,6 +502,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 17a85e9b..b68f8060 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -416,6 +418,27 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" end +// Rule unit: X.9 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + + +// Rule unit: X.10 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------ // Rule unit: FA.1 diff --git a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java index e8bb5be2..7d1aa705 100644 --- a/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java +++ b/redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java @@ -33,7 +33,11 @@ public class RuleFileMigrator { rulesToAdd.forEach(ruleFileBluePrint::addRule); } - String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint); + RuleFileBluePrint newBluePrint = new RuleFileBluePrint(combinedBluePrint.imports(), + combinedBluePrint.globals(), + combinedBluePrint.queries(), + ruleFileBluePrint.ruleClasses()); + String migratedRulesString = RuleFileFactory.buildRuleString(newBluePrint); String migratedFilePath = ruleFile.getAbsolutePath(); try (var out = new FileOutputStream(migratedFilePath)) { out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8)); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index 57124c27..ee0481dc 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1597,6 +1599,24 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end +// Rule unit: X.9.0 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + +// Rule unit: X.10.0 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end //------------------------------------ File attributes rules ------------------------------------ diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 2bf9e43e..d7c48c46 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -54,6 +54,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine; + global Document document global EntityCreationService entityCreationService global ManualChangesApplicationService manualChangesApplicationService @@ -1299,7 +1301,6 @@ rule "TAB.7.0: Indicator (Species)" .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found")); end - //------------------------------------ Manual redaction rules ------------------------------------ // Rule unit: MAN.0 @@ -1561,6 +1562,26 @@ rule "X.8.1: Remove Entity when intersected by imported Entity" retract($other); end +// Rule unit: X.9.0 +rule "X.9.0: Merge mostly contained signatures" + when + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI) + $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8)) + then + $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature"); + $signature.addEngine(LayoutEngine.AI); + end + +// Rule unit: X.10.0 +rule "X.10.0: remove false positives of ai" + when + $anyImage: Image(engines contains LayoutEngine.ALGORITHM) + $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8)) + then + $aiSignature.remove("X.10.0", "Removed because false positive"); + end + + //------------------------------------ File attributes rules ------------------------------------