Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8690: Overlapping SKIPPED and APPLIED of same type

Browse Source
This commit is contained in:
Kilian Schüttler 2024-04-10 14:52:04 +02:00
parent a16ffb3b95
commit ce596adc7e
11 changed files with 90 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -1459,7 +1459,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1594,6 +1594,20 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
end end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
end
//------------------------------------ File attributes rules ------------------------------------ //------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1 // Rule unit: FA.1

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -1239,7 +1239,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1247,7 +1247,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

36
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -244,7 +244,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found", "no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)", "Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat( Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(), $section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList() $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
); );
}); });
@ -438,6 +438,19 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end end
rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$table: Table(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: CBI.10 // Rule unit: CBI.10
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@ -1358,6 +1371,24 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
@ -1861,7 +1892,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1869,7 +1900,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1314,7 +1314,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -798,7 +798,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -806,7 +806,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

5
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -187,7 +187,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found", "no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)", "Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat( Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(), $section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList() $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
); );
}); });
@ -1327,7 +1327,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -1335,7 +1335,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -250,7 +250,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -258,7 +258,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -376,7 +376,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -276,7 +276,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
@ -284,7 +284,6 @@ rule "X.0.0: Remove Entity contained by Entity of same type"
end end
// Rule unit: X.2 // Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64 salience 64

35
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -244,7 +244,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
"no_redaction_indicator but also redaction_indicator found", "no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)", "Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat( Stream.concat(
$section.getEntitiesOfType("vertebrate").stream(), $section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList() $section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
); );
}); });
@ -438,6 +438,18 @@ rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebr
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)")); .forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end end
rule "CBI.9.3: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$table: Table(hasHeader("Author"))
then
$table.streamTableCellsWithHeader("Author")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.3", "Author found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
// Rule unit: CBI.10 // Rule unit: CBI.10
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)" rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
@ -1357,6 +1369,25 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)")); .forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE")) $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
@ -1856,7 +1887,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");

2
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1459,7 +1459,7 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type" rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65 salience 65
when when
$larger: TextEntity($type: type(), $entityType: entityType, active()) $larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges()) $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type"); $contained.remove("X.0.0", "remove Entity contained by Entity of same type");