Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8690: Overlapping SKIPPED and APPLIED of same type

Browse Source
This commit is contained in:
Kilian Schüttler 2024-04-10 14:52:26 +02:00
parent 7a9b1c65d7
commit a2e72f8a62
10 changed files with 83 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -901,6 +901,14 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
@ -911,14 +919,6 @@ rule "PII.10.0: Redact study director abbreviation"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
@ -1229,7 +1229,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1237,7 +1237,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

36
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -242,7 +242,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(),
$section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
);
});
@ -436,6 +436,19 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$table: Table(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: CBI.10
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@ -1356,6 +1369,24 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
@ -1832,7 +1863,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1840,7 +1871,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1312,7 +1312,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -796,7 +796,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -804,7 +804,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -185,7 +185,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(),
$section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
);
});
@ -815,7 +815,6 @@ rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
.forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1318,7 +1317,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1326,7 +1325,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -248,7 +248,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -256,7 +256,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -374,7 +374,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -274,7 +274,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -282,7 +282,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64

35
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -243,7 +243,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(),
$section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
);
});
@ -437,6 +437,18 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$table: Table(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: CBI.10
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@ -1356,6 +1368,25 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
@ -1855,7 +1886,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");

2
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1458,7 +1458,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, active())
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");