From e655ee3bb9239f4c13e4659bad829f85a64dcc30 Mon Sep 17 00:00:00 2001 From: Ali Oezyetimoglu Date: Fri, 1 Dec 2023 11:50:23 +0100 Subject: [PATCH] RED-8007: made "Vertebrate == Yes" more safe --- .../resources/drools/acceptance_rules.drl | 48 +++++----- .../drools/all_redact_manager_rules.drl | 94 +++++++++---------- .../src/test/resources/drools/rules.drl | 58 ++++++------ .../src/test/resources/drools/rules_v2.drl | 4 +- .../resources/all_redact_manager_rules.drl | 94 +++++++++---------- 5 files changed, 149 insertions(+), 149 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index c98397d2..3f3c7266 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -84,7 +84,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" // Rule unit: CBI.0 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -92,7 +92,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -102,7 +102,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" // Rule unit: CBI.1 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); @@ -110,7 +110,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -159,7 +159,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -172,7 +172,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -187,7 +187,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -200,7 +200,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -226,7 +226,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -239,7 +239,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -272,7 +272,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -285,7 +285,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -301,7 +301,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -309,7 +309,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -319,7 +319,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)" // Rule unit: PII.1 rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -328,7 +328,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -339,7 +339,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" // Rule unit: PII.2 rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -357,7 +357,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -377,7 +377,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -386,7 +386,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -409,7 +409,7 @@ rule "ETC.0.0: Purity Hint" // Rule unit: ETC.2 rule "ETC.2.0: Redact signatures (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -417,7 +417,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" rule "ETC.2.1: Redact signatures (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -427,7 +427,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" // Rule unit: ETC.3 rule "ETC.3.0: Skip logos (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.skip("ETC.3.0", "Logo Found"); @@ -435,7 +435,7 @@ rule "ETC.3.0: Skip logos (non vertebrate study)" rule "ETC.3.1: Redact logos (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 8436fd68..6810853f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -71,7 +71,7 @@ query "getFileAttributes" // Rule unit: SYN.0 rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("CTL/") || containsString("BL/")) then Stream.concat( @@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" // Rule unit: CBI.0 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" // Rule unit: CBI.1 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); @@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -375,7 +375,7 @@ rule "CBI.8.1: Redacted because table row contains must_redact entity" rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -388,7 +388,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -403,7 +403,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -416,7 +416,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -442,7 +442,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -461,7 +461,7 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V // Rule unit: CBI.13 rule "CBI.13.0: Ignore CBI Address recommendations" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); @@ -561,7 +561,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -574,7 +574,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -640,7 +640,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -653,7 +653,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -667,7 +667,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC // Rule unit: CBI.21 rule "CBI.21.0: Redact short Authors section (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() then @@ -679,7 +679,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)" rule "CBI.21.1: Redact short Authors section (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() then @@ -693,7 +693,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)" // Rule unit: CBI.22 rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in non-vertebrate documents" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Vertebrate study Y/N"), hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) @@ -707,7 +707,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -715,7 +715,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -725,7 +725,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)" // Rule unit: PII.1 rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -734,7 +734,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -753,7 +753,7 @@ rule "PII.1.2: Redact typoed Emails with indicator" // Rule unit: PII.2 rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -771,7 +771,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -799,7 +799,7 @@ rule "PII.2.2: Redact phone numbers without indicators" // Rule unit: PII.3 rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) @@ -808,7 +808,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) @@ -819,7 +819,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" // Rule unit: PII.4 rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -847,7 +847,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -877,7 +877,7 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) // Rule unit: PII.5 rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -890,7 +890,7 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -905,7 +905,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat // Rule unit: PII.6 rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -917,7 +917,7 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" rule "PII.6.1: Redact line between contact keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -931,7 +931,7 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" // Rule unit: PII.7 rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -949,7 +949,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -969,7 +969,7 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud // Rule unit: PII.8 rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -987,7 +987,7 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -1007,7 +1007,7 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -1016,7 +1016,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -1098,7 +1098,7 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 rule "ETC.2.0: Redact signatures (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -1106,7 +1106,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" rule "ETC.2.1: Redact signatures (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1116,7 +1116,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" // Rule unit: ETC.3 rule "ETC.3.0: Skip logos (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.skip("ETC.3.0", "Logo Found"); @@ -1124,7 +1124,7 @@ rule "ETC.3.0: Skip logos (non vertebrate study)" rule "ETC.3.1: Redact logos (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1193,7 +1193,7 @@ rule "ETC.7.0: Guidelines FileAttributes" // Rule unit: ETC.8 rule "ETC.8.0: Redact formulas (vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -1201,7 +1201,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" rule "ETC.8.1: Redact formulas (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1211,7 +1211,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" // Rule unit: ETC.9 rule "ETC.9.0: Redact skipped impurities" when - FileAttribute(label == "Redact Skipped Impurities", value == "Yes") + FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y") $skippedImpurities: TextEntity(type == "skipped_impurities") then $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); @@ -1219,7 +1219,7 @@ rule "ETC.9.0: Redact skipped impurities" rule "ETC.9.1: Redact impurities" when - FileAttribute(label == "Redact Impurities", value == "Yes") + FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y") $skippedImpurities: TextEntity(type == "impurities") then $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 18d1ce82..6b0a5806 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -71,7 +71,7 @@ query "getFileAttributes" // Rule unit: SYN.0 rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("CTL/") || containsString("BL/")) then Stream.concat( @@ -250,7 +250,7 @@ rule "CBI.8.1: Redacted because table row contains must_redact entity" rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -263,7 +263,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -289,7 +289,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -308,7 +308,7 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -397,7 +397,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -410,7 +410,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -476,7 +476,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -489,7 +489,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -505,7 +505,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -513,7 +513,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -523,7 +523,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)" // Rule unit: PII.1 rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -532,7 +532,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -543,7 +543,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" // Rule unit: PII.4 rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -571,7 +571,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -601,7 +601,7 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) // Rule unit: PII.6 rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -613,7 +613,7 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" rule "PII.6.1: Redact line between contact keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -627,7 +627,7 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" // Rule unit: PII.7 rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -645,7 +645,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -665,7 +665,7 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud // Rule unit: PII.8 rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -683,7 +683,7 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -703,7 +703,7 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -712,7 +712,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -755,7 +755,7 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 rule "ETC.2.0: Redact signatures (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -763,7 +763,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" rule "ETC.2.1: Redact signatures (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -773,7 +773,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" // Rule unit: ETC.3 rule "ETC.3.0: Skip logos (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.skip("ETC.3.0", "Logo Found"); @@ -781,7 +781,7 @@ rule "ETC.3.0: Skip logos (non vertebrate study)" rule "ETC.3.1: Redact logos (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -836,7 +836,7 @@ rule "ETC.7.0: Guidelines FileAttributes" // Rule unit: ETC.8 rule "ETC.8.0: Redact formulas (vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -844,7 +844,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" rule "ETC.8.1: Redact formulas (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 08734f00..30ecca35 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -71,7 +71,7 @@ query "getFileAttributes" // Rule unit: CBI.0 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -83,7 +83,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index d696a4db..63749516 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -71,7 +71,7 @@ query "getFileAttributes" // Rule unit: SYN.0 rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("CTL/") || containsString("BL/")) then Stream.concat( @@ -97,7 +97,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" // Rule unit: CBI.0 rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -105,7 +105,7 @@ rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)" rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_author", dictionaryEntry) then $entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -115,7 +115,7 @@ rule "CBI.0.1: Redact CBI Authors (vertebrate Study)" // Rule unit: CBI.1 rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study"); @@ -123,7 +123,7 @@ rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.1: Redact CBI Address (vertebrate Study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", dictionaryEntry) then $entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -375,7 +375,7 @@ rule "CBI.8.1: Redacted because table row contains must_redact entity" rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -388,7 +388,7 @@ rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebr rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -403,7 +403,7 @@ rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)")) then $table.streamTableCellsWithHeader("Author(s)") @@ -416,7 +416,7 @@ rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author")) then $table.streamTableCellsWithHeader("Author") @@ -442,7 +442,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -461,7 +461,7 @@ rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() @@ -493,7 +493,7 @@ rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'V // Rule unit: CBI.13 rule "CBI.13.0: Ignore CBI Address recommendations" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $entity: TextEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION) then $entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations"); @@ -561,7 +561,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -574,7 +574,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)" rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("et al.")) then entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) @@ -640,7 +640,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -653,7 +653,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" agenda-group "LOCAL_DICTIONARY_ADDS" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) then entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) @@ -666,7 +666,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC // Rule unit: CBI.21 rule "CBI.21.0: Redact short Authors section (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() then @@ -678,7 +678,7 @@ rule "CBI.21.0: Redact short Authors section (non vertebrate study)" rule "CBI.21.1: Redact short Authors section (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsAnyStringIgnoreCase("author(s)", "author", "authors"), length() < 50, getTreeId().get(0) <= 20) //TODO: evaluate the reason of this rule not TextEntity(type == "CBI_author", engines contains Engine.NER) from $section.getEntities() then @@ -692,7 +692,7 @@ rule "CBI.21.1: Redact short Authors section (vertebrate study)" // Rule unit: CBI.22 rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in non-vertebrate documents" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $table: Table(hasHeader("Vertebrate study Y/N"), hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) then $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) @@ -706,7 +706,7 @@ rule "CBI.22.0: Redact Addresses in Reference Tables for vertebrate studies in n // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -714,7 +714,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $pii: TextEntity(type == "PII", dictionaryEntry) then $pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -724,7 +724,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)" // Rule unit: PII.1 rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -733,7 +733,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("@")) then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) @@ -752,7 +752,7 @@ rule "PII.1.2: Redact typoed Emails with indicator" // Rule unit: PII.2 rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -770,7 +770,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsString("Contact") || containsString("Telephone") || containsString("Phone") || @@ -798,7 +798,7 @@ rule "PII.2.2: Redact phone numbers without indicators" // Rule unit: PII.3 rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) @@ -807,7 +807,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(matchesRegex("[+]\\d{1,}")) then entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) @@ -817,7 +817,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" // Rule unit: PII.4 rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -845,7 +845,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st rule "PII.4.1: Redact line after contact information keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -875,7 +875,7 @@ rule "PII.4.1: Redact line after contact information keywords (vertebrate study) // Rule unit: PII.5 rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -888,7 +888,7 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $contactKeyword: String() from List.of("Contact point:", "Contact:", "Alternative contact:", @@ -903,7 +903,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat // Rule unit: PII.6 rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -915,7 +915,7 @@ rule "PII.6.0: Redact line between contact keywords (non vertebrate study)" rule "PII.6.1: Redact line between contact keywords (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel"))) then Stream.concat( @@ -929,7 +929,7 @@ rule "PII.6.1: Redact line between contact keywords (vertebrate study)" // Rule unit: PII.7 rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -947,7 +947,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(getHeadline().containsString("applicant") || getHeadline().containsString("Primary contact") || getHeadline().containsString("Alternative contact") || @@ -967,7 +967,7 @@ rule "PII.7.1: Redact contact information if applicant is found (vertebrate stud // Rule unit: PII.8 rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -985,7 +985,7 @@ rule "PII.8.0: Redact contact information if producer is found (non vertebrate s rule "PII.8.1: Redact contact information if producer is found (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $section: Section(containsStringIgnoreCase("producer of the plant protection") || containsStringIgnoreCase("producer of the active substance") || containsStringIgnoreCase("manufacturer of the active substance") || @@ -1005,7 +1005,7 @@ rule "PII.8.1: Redact contact information if producer is found (vertebrate study // Rule unit: PII.9 rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -1014,7 +1014,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) then entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document) @@ -1095,7 +1095,7 @@ rule "ETC.1.0: Redact Purity" // Rule unit: ETC.2 rule "ETC.2.0: Redact signatures (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -1103,7 +1103,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)" rule "ETC.2.1: Redact signatures (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $signature: Image(imageType == ImageType.SIGNATURE) then $signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1113,7 +1113,7 @@ rule "ETC.2.1: Redact signatures (vertebrate study)" // Rule unit: ETC.3 rule "ETC.3.0: Skip logos (non vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.skip("ETC.3.0", "Logo Found"); @@ -1121,7 +1121,7 @@ rule "ETC.3.0: Skip logos (non vertebrate study)" rule "ETC.3.1: Redact logos (vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.LOGO) then $logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1190,7 +1190,7 @@ rule "ETC.7.0: Guidelines FileAttributes" // Rule unit: ETC.8 rule "ETC.8.0: Redact formulas (vertebrate study)" when - not FileAttribute(label == "Vertebrate Study", value == "Yes") + not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); @@ -1198,7 +1198,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)" rule "ETC.8.1: Redact formulas (non vertebrate study)" when - FileAttribute(label == "Vertebrate Study", value == "Yes") + FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") $logo: Image(imageType == ImageType.FORMULA) then $logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); @@ -1207,7 +1207,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)" // Rule unit: ETC.9 rule "ETC.9.0: Redact skipped impurities" when - FileAttribute(label == "Redact Skipped Impurities", value == "Yes") + FileAttribute(label == "Redact Skipped Impurities", value soundslike "Yes" || value.toLowerCase() == "y") $skippedImpurities: TextEntity(type == "skipped_impurities") then $skippedImpurities.redact("ETC.9.0", "Occasional Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009"); @@ -1215,7 +1215,7 @@ rule "ETC.9.0: Redact skipped impurities" rule "ETC.9.1: Redact impurities" when - FileAttribute(label == "Redact Impurities", value == "Yes") + FileAttribute(label == "Redact Impurities", value soundslike "Yes" || value.toLowerCase() == "y") $skippedImpurities: TextEntity(type == "impurities") then $skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");