diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java index 402f8e25..c7189e5b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java @@ -42,7 +42,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test - @Disabled public void titleExtraction() throws IOException { AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/A8591B/15-Curacron_ToxicidadeAgudaOral.pdf"); @@ -66,7 +65,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest { @Test - @Disabled public void tableWithEmptyCols() throws IOException { // FIXME TableNodeFactory: 36, why has table no rows/cols here. diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index ff014963..e44ff68b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -78,9 +78,7 @@ rule "H.0.0: Study Type File Attribute" end - - -rule "1: Guidelines" +rule "DOC.1.0: Guidelines" when $section: Section( ( @@ -93,36 +91,60 @@ rule "1: Guidelines" || containsString("EPA") || containsString("OPPTS") ) - ) + ) then - Stream.of(entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("OECD Guideline found")), - entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("OECD Guideline found")), - entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("EPA Guideline found")), - entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("EC Guideline found")), - entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("EC Guideline found")), - entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("OECD Guideline year found")), - // Examples found in PoC 1 - //entityCreationService.byRegex("((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\.)|(OECD Test Guideline)|(OECD \\[Test Guideline, Number)) \\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("OECD Guideline year found")), - entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("EPA Guideline found")), - // new approach OECD Guideline - entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).peek(e -> e.setRedactionReason("OECD Guideline no. found")), - entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $section).peek(e -> e.setRedactionReason("OECD Guideline year found")), - entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $section).peek(e -> e.setRedactionReason("OECD Guideline year found")), - entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $section).peek(e -> e.setRedactionReason("OECD Guideline number found")), - entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $section).peek(e -> e.setRedactionReason("OECD Guideline year found")), - // missing OECD guideline rules for RFP demo file - entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).peek(e -> e.setRedactionReason("OECD Guideline number found")), - entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $section).peek(e -> e.setRedactionReason("OECD Guideline found"))) - .flatMap(a -> a) - .forEach(e -> { - e.addMatchedRule("1"); - e.setRedaction(true); - e.setLegalBasis("n-a"); - }); + entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline found", "n-a") + ); + entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline found", "n-a") + ); + entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "EPA Guideline found", "n-a") + ); + entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "EC Guideline found", "n-a") + ); + entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "EC Guideline found", "n-a") + ); + entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline year found", "n-a") + ); +// Examples found in PoC 1 +// entityCreationService.byRegex("((OECD Guidelines for Testing of Chemicals, Procedure)|(OECD Guidelines for the Testing of Chemicals No\\.)|(OECD Test Guideline)|(OECD \\[Test Guideline, Number)) \\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity -> +// entity.apply("DOC.1.0", "OECD Guideline year found", "n-a") +// ); + entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "EPA Guideline found", "n-a") + ); +// new approach OECD Guideline + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline no. found", "n-a") + ); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline year found", "n-a") + ); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline year found", "n-a") + ); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline number found", "n-a") + ); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline year found", "n-a") + ); +// missing OECD guideline rules for RFP demo file + entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline number found", "n-a") + ); + entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity -> + entity.apply("DOC.1.0", "OECD Guideline found", "n-a") + ); end -rule "1b: Guidelines" +rule "DOC.1.2: Guidelines" when $section: Section( ( @@ -140,21 +162,17 @@ rule "1b: Guidelines" || hasEntitiesOfType("epa_guideline") || hasEntitiesOfType("ec_guideline") ) - ) + ) then - $section.getEntitiesOfType(List.of("oecd_guideline","ec_guideline", "epa_guideline")) - .forEach(entity -> { - entity.setRedactionReason("OECD guideline found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("1"); - }); + $section.getEntitiesOfType(List.of("oecd_guideline","ec_guideline", "epa_guideline")).forEach(entity -> { + entity.apply("DOC.1.2", "OECD guideline found.", "n-a"); + }); end -rule "1c: Guidelines" +rule "DOC.1.3: Guidelines" when - $section: Section( + $section: Section( ( hasEntitiesOfType("oecd_guideline") || hasEntitiesOfType("epa_guideline") @@ -172,7 +190,7 @@ rule "1c: Guidelines" || containsString("OPPTS") ) ) - ) + ) then $section.getEntitiesOfType(List.of("oecd_guideline", "ec_guideline", "epa_guideline")).forEach(entity -> { entity.removeFromGraph(); @@ -181,28 +199,21 @@ rule "1c: Guidelines" end - - -rule "2: Report number" +rule "DOC.2.0: Report number" when $section: Section(containsString("LABORATORY PROJECT ID") , containsString("Report Number:")) then entityCreationService.lineAfterString("Report Number:", "report_number", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { - entity.addMatchedRule("2"); - entity.setRedaction(true); - entity.setRedactionReason("Report number found"); - entity.setLegalBasis("n-a"); + entity.apply("DOC.2.0", "Report number found", "n-a"); }); end - - -rule "3: Experimental Starting Date" +rule "DOC.3.0: Experimental Starting Date" when - $section: Section(containsString("Experimental I. Starting Date:") || containsString("Experimental II. Starting Date:") || containsStringIgnoreCase("experimental start date") || containsStringIgnoreCase("experimental starting date")) + $section: Section(containsString("Experimental I. Starting Date:") || containsString("Experimental II. Starting Date:") || containsStringIgnoreCase("experimental start date") || containsStringIgnoreCase("experimental starting date")) then - entityCreationService.lineAfterStrings( + entityCreationService.lineAfterStrings( List.of("Experimental start date", "Experimental start date:", "Experimental Starting Date", @@ -212,20 +223,15 @@ rule "3: Experimental Starting Date" "Experimental Start Date", "Experimental Start Date:", "Experimental I. Starting Date:", - "Experimental II. Starting Date:"), - "experimental_start_date", EntityType.ENTITY, $section) - .forEach(entity -> { - entity.addMatchedRule("3"); - entity.setRedaction(true); - entity.setRedactionReason("Experimental start date found"); - entity.setLegalBasis("n-a"); - }); + "Experimental II. Starting Date:"), "experimental_start_date", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.3.0", "Experimental start date found", "n-a"); + }); end -rule "3: Experimental Completion Date" +rule "DOC.3.1: Experimental Completion Date" when - $section: Section(containsStringIgnoreCase("experimental termination date") || containsStringIgnoreCase("experimental completion date")) + $section: Section(containsStringIgnoreCase("experimental termination date") || containsStringIgnoreCase("experimental completion date")) then entityCreationService.lineAfterStrings( List.of("Experimental termination date", @@ -235,69 +241,57 @@ rule "3: Experimental Completion Date" "Experimental completion date", "Experimental completion date:", "Experimental Termination Date", - "Experimental Termination Date:"), "experimental_end_date", EntityType.ENTITY, $section) - .forEach(entity -> { - entity.addMatchedRule("3"); - entity.setRedaction(true); - entity.setRedactionReason("Experimental end date found"); - entity.setLegalBasis("n-a"); - }); + "Experimental Termination Date:"), "experimental_end_date", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.3.1", "Experimental end date found", "n-a"); + }); end // ignore species and strain in irrelevant study types - rule "4a: Species" + rule "DOC.4.1: Species" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","471","474","487")) $section: Section(hasEntitiesOfType("species") || hasEntitiesOfType("strain")) then $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> { - entity.removeFromGraph(); - retract(entity); - }); + entity.removeFromGraph(); + retract(entity); + }); end // hide all skipped species and strains except in the relevant sections - rule "4b: Species" + rule "DOC.4.2: Species" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436")) - $section: Section( - (hasEntitiesOfType("species") || hasEntitiesOfType("strain")) - && !( - anyHeadlineContainsStringIgnoreCase("test system") - || anyHeadlineContainsStringIgnoreCase("animals") - || anyHeadlineContainsStringIgnoreCase("specification") - ) - ) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436")) + $section: Section( + (hasEntitiesOfType("species") || hasEntitiesOfType("strain")) + && !( + anyHeadlineContainsStringIgnoreCase("test system") + || anyHeadlineContainsStringIgnoreCase("animals") + || anyHeadlineContainsStringIgnoreCase("specification") + ) + ) then $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> { - entity.removeFromGraph(); - retract(entity); - }); + entity.removeFromGraph(); + retract(entity); + }); end -// redact the relevant species -rule "4c: Species" +rule "DOC.4.3: Species" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436")) - $section: Section( - hasEntitiesOfType("species") - ) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436")) + $section: Section(hasEntitiesOfType("species")) then - $section.getEntitiesOfType("species") - .forEach(entity -> { - entity.setRedactionReason("Species found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("25"); - }); + $section.getEntitiesOfType("species").forEach(entity -> { + entity.apply("DOC.4.3", "Species found.", "n-a"); + }); end -// redact the strain -rule "5: Strain" +rule "DOC.5.0: Strain" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436")) $section: Section( @@ -310,17 +304,13 @@ rule "5: Strain" ) ) then - $section.getEntitiesOfType("strain") - .forEach(entity -> { - entity.setRedactionReason("Strain found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("26"); - }); + $section.getEntitiesOfType("strain").forEach(entity -> { + entity.apply("DOC.5.0", "Strain found.", "n-a"); + }); end -//rule "7: study title by document structure" +//rule "DOC.7.0: study title by document structure" // when // $table: Table(isOnPage(1), // (containsString("Final Report") || containsString("SPL")), @@ -329,131 +319,83 @@ rule "5: Strain" // then // // entityCreationService.bySemanticNode($table.getCell(0, 0).streamChildren().toList().get(1), "title", EntityType.ENTITY).ifPresent(entity -> { -// entity.setRedactionReason("Study title found"); -// entity.setLegalBasis("n-a"); -// entity.setRedaction(true); -// entity.addMatchedRule("7"); +// entity.apply("DOC.7.0", "Study title found", "n-a"); // }); // end -rule "7: study title old" +rule "DOC.7.0: study title" when - $section: Section(isOnPage(1) && (containsString("Final Report") || containsString("SPL"))) + $section: Section(isOnPage(1) && (containsString("Final Report") || containsString("SPL"))) then - - entityCreationService.byRegexWithLinebreaks("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $section) - .forEach(entity -> { - entity.setRedactionReason("Title found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("7"); - }); - + entityCreationService.byRegexWithLinebreaks("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { + entity.apply("DOC.7.0", "Title found", "n-a"); + }); entityCreationService.betweenStrings("TITLE", "DATA REQUIREMENT", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { - entity.setRedactionReason("Title found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("7"); - }); + entity.apply("DOC.7.0", "Title found", "n-a"); + }); entityCreationService.betweenStrings("Laboratories", "SPL", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { - entity.setRedactionReason("Title found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("7"); - }); + entity.apply("DOC.7.0", "Title found", "n-a"); + }); end - -rule "8a: Performing Laboratory (Name)" +rule "DOC.8.1: Performing Laboratory (Name)" when $section: Section(containsString("PERFORMING LABORATORY:")) then - entityCreationService.lineAfterString("PERFORMING LABORATORY:", "laboratory_name", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { - entity.setRedactionReason("Performing Laboratory found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("8"); - }); + entityCreationService.lineAfterString("PERFORMING LABORATORY:", "laboratory_name", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> { + entity.apply("DOC.8.1", "Performing Laboratory found", "n-a"); + }); end - rule "8b: Performing Laboratory (Country)" + rule "DOC.8.2: Performing Laboratory (Country)" when nerEntities: NerEntities(hasEntitiesOfType("COUNTRY")) $section: Section(containsString("PERFORMING LABORATORY:")) then nerEntities.streamEntitiesOfType("COUNTRY") + .filter(nerEntity -> $section.getBoundary().contains(nerEntity.boundary())) .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) .forEach(entity -> { - entity.setRedactionReason("Performing Laboratory found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("8"); - insert(entity); - }); + entity.apply("DOC.8.2", "Performing Laboratory found", "n-a"); + insert(entity); + }); end -rule "8c: Performing Laboratory (Country & Name) from dict" +rule "DOC.8.3: Performing Laboratory (Country & Name) from dict" when $section: Section( - ( - hasEntitiesOfType("laboratory_country") - || hasEntitiesOfType("laboratory_name") - ) - && ( - containsString("PERFORMING LABORATORY:") - || ( - containsString("PERFORMING") - && containsString("LABORATORY:") - ) - ) + (hasEntitiesOfType("laboratory_country") || hasEntitiesOfType("laboratory_name")) + && (containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) ) then - $section.getEntitiesOfType("laboratory_country") - .forEach(entity -> { - entity.setRedactionReason("Performing laboratory country dictionary entry found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("8"); - }); - $section.getEntitiesOfType("laboratory_name") - .forEach(entity -> { - entity.setRedactionReason("Performing laboratory name dictionary entry found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("8"); - }); + $section.getEntitiesOfType("laboratory_country").forEach(entity -> { + entity.apply("DOC.8.3", "Performing laboratory country dictionary entry found.", "n-a"); + }); + $section.getEntitiesOfType("laboratory_name").forEach(entity -> { + entity.apply("DOC.8.3", "Performing laboratory name dictionary entry found.", "n-a"); + }); end - rule "8d: Performing Laboratory (Country) from dict" - when - $section: Section( - ( - hasEntitiesOfType("laboratory_country") - || hasEntitiesOfType("laboratory_name") - ) - && !( - containsString("PERFORMING LABORATORY:") - || ( - containsString("PERFORMING") - && containsString("LABORATORY:") - ) - ) - ) - then - $section.getEntitiesOfType(List.of("laboratory_country", "laboratory_name")).forEach(entity -> { - entity.removeFromGraph(); - retract(entity); - }); - end +rule "DOC.8.4: Performing Laboratory (Country) from dict" + when + $section: Section( + (hasEntitiesOfType("laboratory_country") || hasEntitiesOfType("laboratory_name")) + && !(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:"))) + ) + then + $section.getEntitiesOfType(List.of("laboratory_country", "laboratory_name")).forEach(entity -> { + entity.removeFromGraph(); + retract(entity); + }); + end - -rule "9: GLP Study" +rule "DOC.9.0: GLP Study" when $headline: Headline(containsString("GOOD LABORATORY PRACTICE COMPLIANCE") || containsString("GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT") @@ -465,15 +407,12 @@ rule "9: GLP Study" || containsString("STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION")) then entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> { - entity.setRedactionReason("GLP Study found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("9"); - }); + entity.apply("DOC.9.0", "GLP Study found", "n-a"); + }); end -rule "10: Batch number from CoA" +rule "DOC.10.0: Batch number from CoA" when $section: Section( ( @@ -498,25 +437,21 @@ rule "10: Batch number from CoA" ) ) then - entityCreationService.lineAfterStrings(List.of("Batch Identification", - "(Batch Identification):", - "Bateb Identification", - "Batch Wentification", - "Batch Mentification", - "Batch Kientification", - "Barch Identification", - "Bath ldentitfication", - "Batch of test substance :" - ), "batch_number", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Batch number found in CoA"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("10"); - }); + entityCreationService.lineAfterStrings(List.of("Batch Identification", + "(Batch Identification):", + "Bateb Identification", + "Batch Wentification", + "Batch Mentification", + "Batch Kientification", + "Barch Identification", + "Bath ldentitfication", + "Batch of test substance :"), "batch_number", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.10.0", "Batch number found in CoA", "n-a"); + }); end -rule "10a: Batch number" +rule "DOC.10.1: Batch number" when $section: Section( ( @@ -534,28 +469,25 @@ rule "10a: Batch number" && containsStringIgnoreCase("batch") ) then - Stream.of(entityCreationService.byRegex("Batch ID ([A-Z\\d\\-]{7,14})", "batch_number", EntityType.ENTITY, $section) - , entityCreationService.lineAfterStrings(List.of("Batch Identification", - "Batch number:", - "Batch reference number:", - "Batch:", - "Batch/Lot number:", - "Batch (Lot) Number:", - "Batch Number:", - "Batch Nº:", - "Batch no:" - ), "batch_number", EntityType.ENTITY, $section)).flatMap(a -> a) - .forEach(entity -> { - entity.setRedactionReason("Batch number found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("10"); - }); + Stream.of(entityCreationService.byRegex("Batch ID ([A-Z\\d\\-]{7,14})", "batch_number", EntityType.ENTITY, $section), + entityCreationService.lineAfterStrings(List.of("Batch Identification", + "Batch number:", + "Batch reference number:", + "Batch:", + "Batch/Lot number:", + "Batch (Lot) Number:", + "Batch Number:", + "Batch Nº:", + "Batch no:" + ), "batch_number", EntityType.ENTITY, $section)).flatMap(a -> a) + .forEach(entity -> { + entity.apply("DOC.10.1", "Batch number found", "n-a"); + }); end -rule "11: Conclusions - LD50, LC50, Confidence" +rule "DOC.11.0: Conclusions - LD50, LC50, Confidence" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","425","436")) $section: Section( @@ -573,111 +505,64 @@ rule "11: Conclusions - LD50, LC50, Confidence" ) ) then - entityCreationService.byRegexIgnoreCase("(L[D|C]\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)", "ld50_greater", EntityType.ENTITY,2, $section) - .forEach(entity -> { - entity.setRedactionReason("LD50 greater than found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("11"); - }); - - entityCreationService.byRegexIgnoreCase("\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\s?([\\d\\.]{1,6})\\s?mg\\/(?:kg|L)", "ld50_value", EntityType.ENTITY,1, $section) - .forEach(entity -> { - entity.setRedactionReason("LD50 value found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("11"); - }); - - entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_minimal", EntityType.ENTITY,1, $section) - .forEach(entity -> { - entity.setRedactionReason("Minimal Confidence found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("11"); - }); - - entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_maximal", EntityType.ENTITY,2, $section) - .forEach(entity -> { - entity.setRedactionReason("Maximal Confidence found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("11"); - }); + entityCreationService.byRegexIgnoreCase("(L[D|C]\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)", "ld50_greater", EntityType.ENTITY,2, $section).forEach(entity -> { + entity.apply("DOC.11.0", "LD50 greater than found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\s?([\\d\\.]{1,6})\\s?mg\\/(?:kg|L)", "ld50_value", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.11.0", "LD50 value found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_minimal", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.11.0", "Minimal Confidence found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_maximal", EntityType.ENTITY,2, $section).forEach(entity -> { + entity.apply("DOC.11.0", "Maximal Confidence found", "n-a"); + }); end -rule "12: Guideline Deviation" - when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471")) - $section: Section( - ( - getHeadline().containsString("General Information") - || containsString("GENERAL INFORMATION") - ) - && ( - containsStringIgnoreCase("from the") - || containsStringIgnoreCase("to the") - ) - ) - then - - entityCreationService.betweenRegexes("(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?", "(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Deviation from Guidelines found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); - - entityCreationService.betweenRegexes("(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)", "(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Deviation from Study Plan found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); - - entityCreationService.betweenStrings("Deviations from the study plan", "Regulatory Guidelines", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Deviation from the study plan found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); - - entityCreationService.byRegex("(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\.)\\s", "guideline_deviation", EntityType.ENTITY, 1, $section).forEach(entity -> { - entity.setRedactionReason("Guideline deviation found in text."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); - - entityCreationService.betweenStrings("Deviations from the study plan", "validity of the study.", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Deviation from the study plan found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); - end - - -rule "12a: Guideline Deviation in text" +rule "DOC.12.0: Guideline Deviation" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471")) - $section: Section( - getHeadline().containsString("Introduction") - && containsStringIgnoreCase("deviations from the protocol") - ) + $section: Section( + (getHeadline().containsString("General Information") || containsString("GENERAL INFORMATION")) + && (containsStringIgnoreCase("from the") || containsStringIgnoreCase("to the")) + ) then - entityCreationService.byRegex("There were no deviations from the protocol.{1,100}\\.\\s", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Guideline deviation found in text."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("12"); - }); + entityCreationService.betweenRegexes("(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?", "(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.12.0", "Deviation from Guidelines found", "n-a"); + }); + entityCreationService.betweenRegexes("(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)", "(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.12.0", "Deviation from Study Plan found", "n-a"); + }); + entityCreationService.betweenStrings("Deviations from the study plan", "Regulatory Guidelines", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.12.0", "Deviation from the study plan found", "n-a"); + }); + entityCreationService.byRegex("(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\.)\\s", "guideline_deviation", EntityType.ENTITY, 1, $section).forEach(entity -> { + entity.apply("DOC.12.0", "Guideline deviation found in text.", "n-a"); + }); + entityCreationService.betweenStrings("Deviations from the study plan", "validity of the study.", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.12.0", "Deviation from the study plan found", "n-a"); + }); end -rule "13: Clinical Signs" + +rule "DOC.12.1: Guideline Deviation in text" when - FileAttribute(label == "OECD Number", value == "425") + FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471")) + $section: Section( + getHeadline().containsString("Introduction") + && containsStringIgnoreCase("deviations from the protocol") + ) + then + entityCreationService.byRegex("There were no deviations from the protocol.{1,100}\\.\\s", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.12.1", "Guideline deviation found in text.", "n-a"); + }); + end + + +rule "DOC.13.0: Clinical Signs" + when + FileAttribute(label == "OECD Number", value == "425") $section: Section( ( getHeadline().containsString("Clinical Signs") @@ -688,63 +573,41 @@ rule "13: Clinical Signs" ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_signs", EntityType.ENTITY, $section); - entity.setRedactionReason("Clinical Signs found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("13"); + entity.apply("DOC.13.0", "Clinical Signs found", "n-a"); end -rule "14: Dosages" +rule "DOC.14.0: Dosages" when - FileAttribute(label == "OECD Number", value == "425") + FileAttribute(label == "OECD Number", value == "425") $section: Section( - ( - anyHeadlineContainsString("Dosages") - || anyHeadlineContainsString("Study Design") - ) + (anyHeadlineContainsString("Dosages") || anyHeadlineContainsString("Study Design")) && !getHeadline().containsString("TABLE") ) then - entityCreationService.betweenStrings("The animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Dosage found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("14"); - }); - - entityCreationService.betweenStrings("Animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Dosage found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("14"); - }); - - entityCreationService.byRegexWithLinebreaks("(?:\\.[\\s|\\n]|^.{5,20}\\n)([^\\.]{1,200}(?:animal|given|received)[^\\.]{1,200}dose\\s(?:levels?\\s)?(?:of|at)[^\\.]{1,200})(?:\\.[\\s|\\n|$])", "dosages", EntityType.ENTITY,1, $section) - .forEach(entity -> { - entity.setRedactionReason("Dosage found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("14"); -}); + entityCreationService.betweenStrings("The animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.14.0", "Dosage found", "n-a"); + }); + entityCreationService.betweenStrings("Animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.14.0", "Dosage found", "n-a"); + }); + entityCreationService.byRegexWithLinebreaks("(?:\\.[\\s|\\n]|^.{5,20}\\n)([^\\.]{1,200}(?:animal|given|received)[^\\.]{1,200}dose\\s(?:levels?\\s)?(?:of|at)[^\\.]{1,200})(?:\\.[\\s|\\n|$])", "dosages", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.14.0", "Dosage found", "n-a"); + }); end -rule "15: Mortality" + +rule "DOC.15.0: Mortality" when - $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs()) - FileAttribute(label == "OECD Number", value == "425") + $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs()) + FileAttribute(label == "OECD Number", value == "425") then - - var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality", EntityType.ENTITY, $headline.getParent()); - entity.setRedactionReason("Mortality found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("15"); - + var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality", EntityType.ENTITY, $headline.getParent()); + entity.apply("DOC.15.0", "Mortality found", "n-a"); end -rule "17: Study Conclusion" +rule "DOC.17.0: Study Conclusion" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471")) $section: Section( @@ -753,14 +616,11 @@ rule "17: Study Conclusion" ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_conclusion", EntityType.ENTITY, $section); - entity.setRedactionReason("Study Conclusion found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("17"); + entity.apply("DOC.17.0", "Study Conclusion found", "n-a"); end -rule "18: Weight Behavior Changes" +rule "DOC.18.0: Weight Behavior Changes" when FileAttribute(label == "OECD Number", value == "402") $section: Section( @@ -775,13 +635,11 @@ rule "18: Weight Behavior Changes" ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "weight_behavior_changes", EntityType.ENTITY, $section); - entity.setRedactionReason("Weight behavior changes found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("18"); + entity.apply("DOC.18.0", "Weight behavior changes found", "n-a"); end -rule "19: Necropsy findings" + +rule "DOC.19.0: Necropsy findings" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","436")) $section: Section( @@ -796,16 +654,13 @@ rule "19: Necropsy findings" ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "necropsy_findings", EntityType.ENTITY, $section); - entity.setRedactionReason("Necropsy section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("19"); + entity.apply("DOC.19.0", "Necropsy section found", "n-a"); end -rule "22: Clinical observations" +rule "DOC.22.0: Clinical observations" when - FileAttribute(label == "OECD Number", value == "403") + FileAttribute(label == "OECD Number", value == "403") $section: Section( ( anyHeadlineContainsStringIgnoreCase("Clinical Observations") @@ -818,201 +673,165 @@ rule "22: Clinical observations" && hasParagraphs() ) then - - - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_observations", EntityType.ENTITY, $section); - entity.setRedactionReason("Clinical observations section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("22"); + var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "clinical_observations", EntityType.ENTITY, $section); + entity.apply("DOC.22.0", "Clinical observations section found", "n-a"); end /* Die beiden waren vorher auch auskommentiert -rule "23a: Bodyweight changes" +rule "DOC.23.1: Bodyweight changes" when - FileAttribute(label == "OECD Number", value == "403") - $section: Section( + FileAttribute(label == "OECD Number", value == "403") + $section: Section( containsString("Bodyweight") && containsString("Group") ) then - entityCreationService.betweenRegexes("\\.\\s\\bBodyweight\\s", "Group.{0,40}[\\s\\d(.]{1,10}mg/L\\)", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Bodyweight changes found in results"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule(23); + entityCreationService.betweenRegexes("\\.\\s\\bBodyweight\\s", "Group.{0,40}[\\s\\d(.]{1,10}mg/L\\)", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.23.1", "Bodyweight changes found in results", "n-a"); }); entityCreationService.betweenRegexes("\\.\\s\\bBodyweight.{100,500}Group.{0,30}mg/L\\)", "\\..{1,20}Group", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Bodyweight changes found in results"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule(23); + entity.apply("DOC.23.1", "Bodyweight changes found in results", "n-a"); }); end -rule "23b: Bodyweight changes" +rule "DOC.23.2: Bodyweight changes" when - FileAttribute(label == "OECD Number", value == "403") + FileAttribute(label == "OECD Number", value == "403") $section: Section( getHeadline().containsString("Bodyweight") && containsString("Sighting") && containsString("Main") ) then - entityCreationService.betweenRegexes("\\bSighting[\\w\\s]{0,15}[\\s\\d(.]{1,10}mg/L\\)", "\\bMain[\\w\\s]{0,15}[\\s\\(\\d\\.]{1,10}mg/L\\)", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Bodyweight section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule(23); - }); - - entityCreationService.betweenRegexes("\\bMain[\\w\\s]{0,15}[\\s\\d(.]{1,10}mg/L\\)", "the study\\.", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("Bodyweight section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule(23); - }); + entityCreationService.betweenRegexes("\\bSighting[\\w\\s]{0,15}[\\s\\d(.]{1,10}mg/L\\)", "\\bMain[\\w\\s]{0,15}[\\s\\(\\d\\.]{1,10}mg/L\\)", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.23.2", "Bodyweight section found", "n-a"); + }); + entityCreationService.betweenRegexes("\\bMain[\\w\\s]{0,15}[\\s\\d(.]{1,10}mg/L\\)", "the study\\.", "bodyweight_changes", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.23.2", "Bodyweight section found", "n-a"); + }); end */ -rule "23: Bodyweight changes" +rule "DOC.23.0: Bodyweight changes" when - FileAttribute(label == "OECD Number", value == "403") + FileAttribute(label == "OECD Number", value == "403") $section: Section( - ( - getHeadline().containsString("Bodyweight") - || getHeadline().containsString("Bodyweights") - || getHeadline().containsString("Body Weights") - || getHeadline().containsString("Body Weight") + ( + getHeadline().containsString("Bodyweight") + || getHeadline().containsString("Bodyweights") + || getHeadline().containsString("Body Weights") + || getHeadline().containsString("Body Weight") ) && !getHeadline().containsStringIgnoreCase("Appendix") && !getHeadline().containsStringIgnoreCase("TABLE") && hasParagraphs() ) then - var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "bodyweight_changes", EntityType.ENTITY, $section); - entity.setRedactionReason("Bodyweight section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("23"); + var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "bodyweight_changes", EntityType.ENTITY, $section); + entity.apply("DOC.23.0", "Bodyweight section found", "n-a"); end -rule "24: Study Design" +rule "DOC.24.0: Study Design" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","404","405","406","428","429","438","439","474","487")) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","404","405","406","428","429","438","439","474","487")) $section: Section( anyHeadlineContainsStringIgnoreCase("study design") && hasParagraphs() ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "study_design", EntityType.ENTITY, $section); - entity.setRedactionReason("Study design section found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("24"); + entity.apply("DOC.24.0", "Study design section found", "n-a"); end - -rule "25: Results and Conclusion (406, 428, 438, 439, 474 & 487)" +rule "DOC.25.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487")) - $section: Section( - (getHeadline().containsString("Results") || getHeadline().containsString("Conclusion")) - && !getHeadline().containsString("POSITIVE CONTROL") && !getHeadline().containsString("Positive Control") - && !getHeadline().containsString("Evaluation") && !getHeadline().containsString("Micronucleus") && - !getHeadline().containsString("TABLE") && !getHeadline().containsString("DISCUSSION") && - !getHeadline().containsString("CONCLUSIONS") && !getHeadline().containsString("Interpretation") && !getHeadline().containsString("Viability") + FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487")) + $section: Section( + (getHeadline().containsString("Results") || getHeadline().containsString("Conclusion")) + && !getHeadline().containsString("POSITIVE CONTROL") + && !getHeadline().containsString("Positive Control") + && !getHeadline().containsString("Evaluation") + && !getHeadline().containsString("Micronucleus") + && !getHeadline().containsString("TABLE") + && !getHeadline().containsString("DISCUSSION") + && !getHeadline().containsString("CONCLUSIONS") + && !getHeadline().containsString("Interpretation") + && !getHeadline().containsString("Viability") && hasParagraphs() - ) + ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "results_and_conclusion", EntityType.ENTITY, $section); - entity.setRedactionReason("Results and Conclusion found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("25"); + entity.apply("DOC.25.0", "Results and Conclusion found", "n-a"); end -rule "26: Detailing (404 & 405)" +rule "DOC.26.0: Detailing (404 & 405)" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("404","405")) - $section: Section( - getHeadline().containsStringIgnoreCase("Results") && !getHeadline().containsStringIgnoreCase("Evaluation") && !getHeadline().containsStringIgnoreCase("study") && !getHeadline().containsStringIgnoreCase("discussion") && hasParagraphs() - ) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("404","405")) + $section: Section( + getHeadline().containsStringIgnoreCase("Results") + && !getHeadline().containsStringIgnoreCase("Evaluation") + && !getHeadline().containsStringIgnoreCase("study") + && !getHeadline().containsStringIgnoreCase("discussion") + && hasParagraphs() + ) then var paragraphs = $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList(); for(var p : paragraphs){ entityCreationService.bySemanticNode(p, "detailing", EntityType.ENTITY).ifPresent(entity -> { - entity.setRedactionReason("Detailing found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("26"); + entity.apply("DOC.26.0", "Detailing found", "n-a"); }); } end -rule "32: Preliminary Test Results (429)" +rule "DOC.32.0: Preliminary Test Results (429)" when - FileAttribute(label == "OECD Number", value == "429") - $section: Section( - ( - (getHeadline().containsString("Preliminary Screening Test") && containsString("Clinical observations")) - || getHeadline().containsString("Pre-Experiment")) - && hasParagraphs() + FileAttribute(label == "OECD Number", value == "429") + $section: Section( + ((getHeadline().containsString("Preliminary Screening Test") && containsString("Clinical observations")) + || getHeadline().containsString("Pre-Experiment")) + && hasParagraphs() ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "preliminary_test_results", EntityType.ENTITY, $section); - entity.setRedactionReason("Preliminary Test Results found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("32"); + entity.apply("DOC.32.0", "Preliminary Test Results found", "n-a"); end -rule "33: Test Results (429)" +rule "DOC.33.0: Test Results (429)" when - FileAttribute(label == "OECD Number", value == "429") - $section: Section((getHeadline().containsString("RESULTS AND DISCUSSION") || getHeadline().containsString("Estimation of the proliferative response of lymph node cells") || getHeadline().containsString("Results in the Main Experiment")) && hasParagraphs()) + FileAttribute(label == "OECD Number", value == "429") + $section: Section((getHeadline().containsString("RESULTS AND DISCUSSION") || getHeadline().containsString("Estimation of the proliferative response of lymph node cells") || getHeadline().containsString("Results in the Main Experiment")) && hasParagraphs()) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "test_results", EntityType.ENTITY, $section); - entity.setRedactionReason("Test Results found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("33"); + entity.apply("DOC.33.0", "Test Results found", "n-a"); end -rule "34: Approach used (429)" +rule "DOC.34.0: Approach used (429)" when - FileAttribute(label == "OECD Number", value == "429") + FileAttribute(label == "OECD Number", value == "429") $section: Section( hasEntitiesOfType("species") - && ( - containsStringIgnoreCase("animals per") - || containsStringIgnoreCase("animals /") - ) + && (containsStringIgnoreCase("animals per") || containsStringIgnoreCase("animals /")) ) then - - entityCreationService.byRegexIgnoreCase("\\banimals (?:per|\\/) .{0,15}(group)\\b", "approach_used", EntityType.ENTITY,1, $section).forEach(entity -> { - entity.setRedactionReason("Study animal approach found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("34"); - }); + entityCreationService.byRegexIgnoreCase("\\banimals (?:per|\\/) .{0,15}(group)\\b", "approach_used", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.34.0", "Study animal approach found.", "n-a"); + }); end -rule "35: Sex" +rule "DOC.35.0: Sex" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("405","429")) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("405","429")) $section: Section( ( anyHeadlineContainsStringIgnoreCase("animal") @@ -1026,20 +845,16 @@ rule "35: Sex" ) ) then - - entityCreationService.byRegexIgnoreCase("([S|s]ex:)?[\\w\\s]{0,10}\\b(males?|females?)\\b", "sex", EntityType.ENTITY,2, $section).forEach(entity -> { - entity.setRedactionReason("Test animal sex found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); + entityCreationService.byRegexIgnoreCase("([S|s]ex:)?[\\w\\s]{0,10}\\b(males?|females?)\\b", "sex", EntityType.ENTITY,2, $section).forEach(entity -> { + entity.apply("DOC.35.0", "Test animal sex found", "n-a"); + }); end -rule "35a: Animal Number 405" +rule "DOC.35.1: Animal Number 405" when - FileAttribute(label == "OECD Number", value == "405") + FileAttribute(label == "OECD Number", value == "405") $section: Section( ( anyHeadlineContainsStringIgnoreCase("animal") @@ -1053,25 +868,17 @@ rule "35a: Animal Number 405" ) ) then - - entityCreationService.byRegexIgnoreCase("(Number of animals:)[\\w\\s]{0,10}\\b([\\d]{1,3})\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> { - entity.setRedactionReason("Number of animals found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); - - entityCreationService.byRegexIgnoreCase("(?:.{1,10} No\\. )([\\d\\w\\-]{3,8})", "animal_numbers", EntityType.ENTITY,1, $section).forEach(entity -> { - entity.setRedactionReason("Number of animals found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); + entityCreationService.byRegexIgnoreCase("(Number of animals:)[\\w\\s]{0,10}\\b([\\d]{1,3})\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> { + entity.apply("DOC.35.1", "Number of animals found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("(?:.{1,10} No\\. )([\\d\\w\\-]{3,8})", "animal_numbers", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.35.1", "Number of animals found", "n-a"); + }); end -rule "35b: Animal Number 429" +rule "DOC.35.2: Animal Number 429" when FileAttribute(label == "OECD Number", value == "429") $section: Section( @@ -1085,78 +892,54 @@ rule "35b: Animal Number 429" && containsStringIgnoreCase("group") ) then - entityCreationService.byRegexIgnoreCase("(Number of animals per group:)[\\w\\s]{0,10}\\b([\\d]{1,3})\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> { - entity.setRedactionReason("Number of animals in group found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); - entityCreationService.byRegexIgnoreCase("(Number of animals per group:).{0,60}\\b([\\d]{1,3})\\sper group\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> { - entity.setRedactionReason("Number of animals in group found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); - entityCreationService.byRegexIgnoreCase("([\\d]{1,3})[\\w\\s\\/]{0,20}(?:treatment )?group\\b", "number_of_animals", EntityType.ENTITY,1 , $section).forEach(entity -> { - entity.setRedactionReason("Number of animals in group found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("35"); - }); + entity.apply("DOC.35.2", "Number of animals in group found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("(Number of animals per group:).{0,60}\\b([\\d]{1,3})\\sper group\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> { + entity.apply("DOC.35.2", "Number of animals in group found", "n-a"); + }); + entityCreationService.byRegexIgnoreCase("([\\d]{1,3})[\\w\\s\\/]{0,20}(?:treatment )?group\\b", "number_of_animals", EntityType.ENTITY,1 , $section).forEach(entity -> { + entity.apply("DOC.35.2", "Number of animals in group found", "n-a"); + }); end -rule "35c: No. Of animals - Fallback to appendix tables listing all individual animals for 429" +rule "DOC.35.3: No. Of animals - Fallback to appendix tables listing all individual animals for 429" when - $keyword: String() from List.of("Animal Number", "Animal No.", "Animal number") - $table: Table(containsString($keyword) && getHeadline().containsString("TABLE") && getHeadline().containsString("Individual")) - FileAttribute(label == "OECD Number", value == "429") + $keyword: String() from List.of("Animal Number", "Animal No.", "Animal number") + $table: Table(containsString($keyword) && getHeadline().containsString("TABLE") && getHeadline().containsString("Individual")) + FileAttribute(label == "OECD Number", value == "429") then - $table.streamTableCellsWithHeader($keyword) + $table.streamTableCellsWithHeader($keyword) .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "animal_numbers", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.setRedaction(true); - redactionEntity.addMatchedRule("35"); - redactionEntity.setRedactionReason("Animal number found. ("+$keyword+")"); - redactionEntity.setLegalBasis("n-a"); - insert(redactionEntity); + .forEach(entity -> { + entity.apply("DOC.35.3", "Animal number found.", "n-a"); + insert(entity); }); end - -rule "37: 4h Exposure" +rule "DOC.37.0: 4h Exposure" when - FileAttribute(label == "OECD Number", valueEqualsAnyOf("403","436")) + FileAttribute(label == "OECD Number", valueEqualsAnyOf("403","436")) $section: Section( - ( - containsStringIgnoreCase("4 hours") - || containsStringIgnoreCase("four hours") - ) + (containsStringIgnoreCase("4 hours") || containsStringIgnoreCase("four hours")) ) then - /* entityCreationService.byRegexIgnoreCase("(?<=\\.\\s\\b).{1,100}(4|four) hours.{1,250}(?=\\b\\.|\\B\\.)", "4h_exposure", EntityType.ENTITY, $section).forEach(entity -> { - entity.setRedactionReason("4h exposure sentence found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule(37); - });*/ - entityCreationService.byRegexIgnoreCase("((?<=\\.\\s\\b).{1,100}(4|four) hours.*?\\.) ", "4h_exposure", EntityType.ENTITY,1, $section).forEach(entity -> { - entity.setRedactionReason("4h exposure sentence found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("37"); - }); + /* entityCreationService.byRegexIgnoreCase("(?<=\\.\\s\\b).{1,100}(4|four) hours.{1,250}(?=\\b\\.|\\B\\.)", "4h_exposure", EntityType.ENTITY, $section).forEach(entity -> { + entity.apply("DOC.37.0", "4h exposure sentence found", "n-a"); + }); */ + entityCreationService.byRegexIgnoreCase("((?<=\\.\\s\\b).{1,100}(4|four) hours.*?\\.) ", "4h_exposure", EntityType.ENTITY,1, $section).forEach(entity -> { + entity.apply("DOC.37.0", "4h exposure sentence found", "n-a"); + }); end - -rule "39: Dilution of the test substance" +rule "DOC.39.0: Dilution of the test substance" when - FileAttribute(label == "OECD Number", value == "404") + FileAttribute(label == "OECD Number", value == "404") $section: Section( getHeadline().containsString("Formulation") && containsString("dilution") @@ -1164,57 +947,44 @@ rule "39: Dilution of the test substance" ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "dilution", EntityType.ENTITY, $section); - entity.setRedactionReason("Dilution found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("39"); + entity.apply("DOC.39.0", "Dilution found.", "n-a"); end -rule "40: Positive Control" +rule "DOC.40.0: Positive Control" when - FileAttribute(label == "OECD Number", value == "429") + FileAttribute(label == "OECD Number", value == "429") $section: Section( getHeadline().containsStringIgnoreCase("Positive Control") - && !( - getHeadline().containsStringIgnoreCase("Appendix") - || getHeadline().containsStringIgnoreCase("Table") - ) + && !(getHeadline().containsStringIgnoreCase("Appendix") || getHeadline().containsStringIgnoreCase("Table")) && hasParagraphs() ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "positive_control", EntityType.ENTITY, $section); - entity.setRedactionReason("Positive control found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("40"); + entity.apply("DOC.40.0", "Positive control found.", "n-a"); end - -rule "42: Mortality Statement" +rule "DOC.42.0: Mortality Statement" when FileAttribute(label == "OECD Number", value == "402") $headline: Headline(containsString("Mortality") && !containsString("TABLE") && hasParagraphs()) then var entity = entityCreationService.byBoundary(Boundary.merge($headline.getParent().streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "mortality_statement", EntityType.ENTITY, $headline.getParent()); - entity.setRedactionReason("Mortality Statement found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("42"); + entity.apply("DOC.42.0", "Mortality Statement found", "n-a"); end -rule "43: Dose Mortality" +rule "DOC.43.0: Dose Mortality" when - FileAttribute(label == "OECD Number", value == "425") - $table: Table( - (hasHeader("Mortality") || hasHeader("Long Term Results") || hasHeader("Long Term Outcome") || hasHeader("Comments") || hasHeader("Viability / Mortality")) - && - (hasHeader("Dose [mg/kg bodyweight]") || hasHeader("Dose [mg/kg body weight]") ||hasHeader("Dose (mg/kg)") || hasHeader("Dose levei (mg/kg)") || hasHeader("Dose Level (mg/kg)") || hasHeader("Dose level (mg/kg)") || hasHeader("Dosage [mg/kg body weight]")) - ) + FileAttribute(label == "OECD Number", value == "425") + $table: Table( + (hasHeader("Mortality") || hasHeader("Long Term Results") || hasHeader("Long Term Outcome") || hasHeader("Comments") || hasHeader("Viability / Mortality")) + && + (hasHeader("Dose [mg/kg bodyweight]") || hasHeader("Dose [mg/kg body weight]") ||hasHeader("Dose (mg/kg)") || hasHeader("Dose levei (mg/kg)") || hasHeader("Dose Level (mg/kg)") || hasHeader("Dose level (mg/kg)") || hasHeader("Dosage [mg/kg body weight]")) + ) then - Stream.of($table.streamTableCellsWithHeader("Mortality"), + Stream.of($table.streamTableCellsWithHeader("Mortality"), $table.streamTableCellsWithHeader("Comments"), $table.streamTableCellsWithHeader("Long Term Results"), $table.streamTableCellsWithHeader("Long Term Outcome"), @@ -1223,15 +993,12 @@ rule "43: Dose Mortality" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.setRedaction(true); - redactionEntity.addMatchedRule("43"); - redactionEntity.setRedactionReason("Dose Mortality Data found."); - redactionEntity.setLegalBasis("n-a"); - insert(redactionEntity); + .forEach(entity -> { + entity.apply("DOC.43.0", "Dose Mortality Data found.", "n-a"); + insert(entity); }); - Stream.of($table.streamTableCellsWithHeader("Dose [mg/kg bodyweight]"), + Stream.of($table.streamTableCellsWithHeader("Dose [mg/kg bodyweight]"), $table.streamTableCellsWithHeader("Dose [mg/kg body weight]"), $table.streamTableCellsWithHeader("Dose levei (mg/kg)"), $table.streamTableCellsWithHeader("Dose Level (mg/kg)"), @@ -1242,54 +1009,42 @@ rule "43: Dose Mortality" .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY)) .filter(Optional::isPresent) .map(Optional::get) - .forEach(redactionEntity -> { - redactionEntity.setRedaction(true); - redactionEntity.addMatchedRule("43"); - redactionEntity.setRedactionReason("Dose Mortality Data found."); - redactionEntity.setLegalBasis("n-a"); - insert(redactionEntity); + .forEach(entity -> { + entity.apply("DOC.43.0", "Dose Mortality Data found.", "n-a"); + insert(entity); }); end -rule "44: Results (Main Study)" +rule "DOC.44.0: Results (Main Study)" when - FileAttribute(label == "OECD Number", value == "429") + FileAttribute(label == "OECD Number", value == "429") $section: Section( getHeadline().containsString("Results") - && getHeadline().getTextBlock().toString().length() < 20 + && getHeadline().getBoundary().length() < 20 && hasParagraphs() - && !( - getHeadline().containsString("Appendix") - || getHeadline().containsString("Table") - ) + && !(getHeadline().containsString("Appendix") || getHeadline().containsString("Table")) ) then - var paragraphs = $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList(); - for(var p : paragraphs){ - entityCreationService.bySemanticNode(p, "results_(main_study)", EntityType.ENTITY).ifPresent(entity -> { - entity.setRedactionReason("Results for main study found."); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("44"); - }); - } + var paragraphs = $section.streamAllSubNodesOfType(NodeType.PARAGRAPH).toList(); + for(var p : paragraphs){ + entityCreationService.bySemanticNode(p, "results_(main_study)", EntityType.ENTITY).ifPresent(entity -> { + entity.apply("DOC.44.0", "Results for main study found.", "n-a"); + }); + } end -rule "45: Doses (mg/kg bodyweight)" +rule "DOC.45.0: Doses (mg/kg bodyweight)" when - FileAttribute(label == "OECD Number", value == "402") + FileAttribute(label == "OECD Number", value == "402") $section: Section( anyHeadlineContainsStringIgnoreCase("study design") && hasParagraphs() ) then var entity = entityCreationService.byBoundary(Boundary.merge($section.streamAllSubNodesOfType(NodeType.PARAGRAPH).map(SemanticNode::getBoundary).toList()), "doses_(mg_kg_bw)", EntityType.ENTITY, $section); - entity.setRedactionReason("Doses per bodyweight information found"); - entity.setLegalBasis("n-a"); - entity.setRedaction(true); - entity.addMatchedRule("45"); + entity.apply("DOC.45.0", "Doses per bodyweight information found", "n-a"); end @@ -1338,8 +1093,7 @@ rule "MAN.2.0: Apply force redaction" ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.setLegalBasis($legalBasis); - $entityToForce.setRedaction(true); + $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); end