diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 4df6c28f..b6ad71dd 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -901,6 +901,14 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" + when + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); + end + // Rule unit: PII.10 rule "PII.10.0: Redact study director abbreviation" @@ -911,14 +919,6 @@ rule "PII.10.0: Redact study director abbreviation" .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); end -rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" - when - $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) - then - entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) - .forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); - end - // Rule unit: PII.11 rule "PII.11.0: Redact On behalf of Sequani Ltd.:" @@ -1229,7 +1229,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -1237,7 +1237,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 4ff8dd5e..e3ba12ab 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -242,7 +242,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $section.getEntitiesOfType("vertebrate").stream(), + $section.getEntitiesOfType("redaction_indicator").stream(), $section.getEntitiesOfType("no_redaction_indicator").stream()).toList() ); }); @@ -436,6 +436,19 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); end +rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); + end + // Rule unit: CBI.10 rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" @@ -1356,6 +1369,24 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" when $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) @@ -1832,7 +1863,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -1840,7 +1871,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 43e5c446..a51f1b51 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -1312,7 +1312,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl index 79e9acaa..ee3f0775 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl @@ -796,7 +796,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -804,7 +804,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index cf20b9be..5333c2d1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -185,7 +185,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $section.getEntitiesOfType("vertebrate").stream(), + $section.getEntitiesOfType("redaction_indicator").stream(), $section.getEntitiesOfType("no_redaction_indicator").stream()).toList() ); }); @@ -815,7 +815,6 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)" .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002")); end - rule "PII.6.3: Redact line between contact keywords (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") @@ -1318,7 +1317,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -1326,7 +1325,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 01279a69..c79486b0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -248,7 +248,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -256,7 +256,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index dfe2e86b..35db9f74 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -374,7 +374,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl index 29bfbe84..a2a68466 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -274,7 +274,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); @@ -282,7 +282,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type" end - // Rule unit: X.2 rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" salience 64 diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index d723e2ac..cf25940a 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -243,7 +243,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red "no_redaction_indicator but also redaction_indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)", Stream.concat( - $section.getEntitiesOfType("vertebrate").stream(), + $section.getEntitiesOfType("redaction_indicator").stream(), $section.getEntitiesOfType("no_redaction_indicator").stream()).toList() ); }); @@ -437,6 +437,18 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); end +rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $table: Table(hasHeader("Author")) + then + $table.streamTableCellsWithHeader("Author") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); + end // Rule unit: CBI.10 rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" @@ -1356,6 +1368,25 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); end +rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" + when + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); + end + +rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" + when + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") + $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) + then + entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) + .forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002")); + end + + rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" when $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) @@ -1855,7 +1886,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl index 02892ba6..705daa4e 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl @@ -1458,7 +1458,7 @@ rule "MAN.4.1: Apply legal basis change" rule "X.0.0: Remove Entity contained by Entity of same type" salience 65 when - $larger: TextEntity($type: type(), $entityType: entityType, active()) + $larger: TextEntity($type: type(), $entityType: entityType, !removed()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) then $contained.remove("X.0.0", "remove Entity contained by Entity of same type");