Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6009: Document Tree Structure

Browse Source
This commit is contained in:
Kilian Schüttler 2023-06-20 16:58:59 +02:00
parent 778286e3b8
commit ad0b0f3f8d
6 changed files with 155 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
View File

@ -96,6 +96,12 @@ public class RedactionEntity {
}
public boolean isActive() {
return !ignored && !removed;
}
public void addIntersectingNode(SemanticNode containingNode) {
intersectingNodes.add(containingNode);

12
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
View File

@ -188,9 +188,9 @@ public interface SemanticNode {
* @param type string representing the type of entity to check for
* @return true, if this SemanticNode has at least one Entity of the provided type
*/
default boolean hasEntitiesOfType(String type) {
default boolean hasActiveEntitiesOfType(String type) {
return getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
}
@ -200,9 +200,9 @@ public interface SemanticNode {
* @param type string representing the type of entities to return
* @return List of RedactionEntities of any the type
*/
default List<RedactionEntity> getEntitiesOfType(String type) {
default List<RedactionEntity> getActiveEntitiesOfType(String type) {
return getEntities().stream().filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
}
@ -212,9 +212,9 @@ public interface SemanticNode {
* @param types A list of strings representing the types of entities to return
* @return List of RedactionEntities of any provided type
*/
default List<RedactionEntity> getEntitiesOfType(List<String> types) {
default List<RedactionEntity> getActiveEntitiesOfType(List<String> types) {
return getEntities().stream().filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
}

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java
View File

@ -107,9 +107,10 @@ public class Table implements SemanticNode {
* @param types type strings to check whether a row contains an entity like them
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
*/
public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
public Stream<RedactionEntity> streamEntitiesWhereRowContainsActiveEntitiesOfType(List<String> types) {
List<Integer> rowsWithEntityOfType = getEntities().stream()
.filter(RedactionEntity::isActive)
.filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
.map(RedactionEntity::getIntersectingNodes)
.filter(node -> node instanceof TableCell)
@ -127,11 +128,14 @@ public class Table implements SemanticNode {
* @param types type strings to check whether a row contains an entity like them
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
*/
public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List<String> types) {
return IntStream.range(0, numberOfRows)
.boxed()
.filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).noneMatch(entity -> types.contains(entity.getType())))
.filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities)
.flatMap(Collection::stream)
.filter(RedactionEntity::isActive)
.noneMatch(entity -> types.contains(entity.getType())))
.flatMap(this::streamRow)
.map(TableCell::getEntities)
.flatMap(Collection::stream);
@ -273,14 +277,14 @@ public class Table implements SemanticNode {
* @param redactionEntity the entity, which appears in the row to search
* @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
*/
public List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
public List<RedactionEntity> getActiveEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
return redactionEntity.getIntersectingNodes()
.stream()
.filter(node -> node instanceof TableCell)
.map(node -> (TableCell) node)
.flatMap(tableCellNode -> streamRow(tableCellNode.getRow()))
.map(cell -> cell.getEntitiesOfType(type))
.map(cell -> cell.getActiveEntitiesOfType(type))
.flatMap(Collection::stream)
.toList();
}

184
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -74,24 +74,24 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
// Rule unit: CBI.3
rule "CBI.3.0: Redacted because Section contains Vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addReferences($section.getEntitiesOfType("vertebrate"));
entity.addReferences($section.getActiveEntitiesOfType("vertebrate"));
});
end
rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
when
$table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$table: Table(hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate found");
@ -99,15 +99,15 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
});
end
rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), !hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("No vertebrate found");
entity.setRedaction(false);
@ -118,9 +118,9 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
$table: Table(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))
then
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
$table.streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("No vertebrate found");
@ -135,35 +135,35 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate but a no redaction indicator found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(4);
entity.addReferences($section.getEntitiesOfType("no_redaction_indicator"));
entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator"));
});
end
rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$table: Table(hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate but a no redaction indicator found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(4);
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
});
end
@ -172,29 +172,29 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("redaction_indicator"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
hasActiveEntitiesOfType("redaction_indicator"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(5);
entity.addReferences($section.getEntitiesOfType("no_redaction_indicator"));
entity.addReferences($section.getEntitiesOfType("redaction_indicator"));
entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator"));
entity.addReferences($section.getActiveEntitiesOfType("redaction_indicator"));
});
end
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$table: Table(hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator"))
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found");
@ -202,8 +202,8 @@ rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also red
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(5);
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
});
end
@ -211,24 +211,24 @@ rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also red
// Rule unit: CBI.8
rule "CBI.8.0: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("must_redact entity found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(15);
entity.addReferences($section.getEntitiesOfType("must_redact"));
entity.addReferences($section.getActiveEntitiesOfType("must_redact"));
});
end
rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
when
$table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$table: Table(hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("must_redact"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("must_redact entity found");
@ -236,7 +236,7 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(15);
entity.addReferences($table.getEntitiesOfTypeInSameRow("must_redact", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("must_redact", entity));
});
end
@ -290,7 +290,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
when
$table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
then
$table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
$table.getActiveEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
@ -375,7 +375,7 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
.forEach(keywordEntity -> insert(keywordEntity));
$section.getEntitiesOfType(List.of($keyword, $residueKeyword))
$section.getActiveEntitiesOfType(List.of($keyword, $residueKeyword))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(15);
@ -507,13 +507,50 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
end
// Rule unit: CBI.20
rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(false);
laboratoryEntity.addMatchedRule(20);
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(true);
laboratoryEntity.addMatchedRule(20);
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found");
laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
//------------------------------------ PII rules ------------------------------------
// Rule unit: PII.0
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: RedactionEntity(type == "PII", redaction == false)
$pii: RedactionEntity(type == "PII", dictionaryEntry)
then
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
@ -524,7 +561,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
rule "PII.0.1: Redact all PII (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: RedactionEntity(type == "PII", redaction == false)
$pii: RedactionEntity(type == "PII", dictionaryEntry)
then
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
@ -596,6 +633,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(4);
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
@ -630,6 +668,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(4);
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
@ -650,6 +689,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(6);
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
@ -668,6 +708,7 @@ rule "PII.6.1: redact line between contact keywords"
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(6);
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
@ -845,66 +886,13 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
end
// Rule unit: PII.10
rule "PII.10.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(false);
laboratoryEntity.addMatchedRule(10);
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
rule "PII.10.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(true);
laboratoryEntity.addMatchedRule(10);
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found");
laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
// Rule unit: PII.12
rule "PII.12.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(12);
authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
end
// Rule unit: PII.13
rule "PII.13.0: Expand PII entities with salutation prefix"
rule "PII.12.0: Expand PII entities with salutation prefix"
when
$entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
expandedEntity.addMatchedRule(13);
expandedEntity.addMatchedRule(12);
insert(expandedEntity);
end

48
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl
View File

@ -79,10 +79,10 @@ rule "0: Expand CBI_author and PII entities with salutation prefix"
rule "1: Redacted because Section contains Vertebrate"
when
$section: Section(hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(1);
@ -93,10 +93,10 @@ rule "1: Redacted because Section contains Vertebrate"
rule "2: Not Redacted because Section contains no Vertebrate"
when
$section: Section(!hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(!hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(2);
@ -106,11 +106,11 @@ rule "2: Not Redacted because Section contains no Vertebrate"
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(3);
@ -120,12 +120,12 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(4);
@ -137,12 +137,12 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
rule "5: Do not redact Names and Addresses if published information found"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("published_information"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
List<RedactionEntity> publishedInformationEntities = $section.getActiveEntitiesOfType("published_information");
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
@ -193,10 +193,10 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate
rule "8: Redact if must_redact entity is found"
when
$section: Section(hasEntitiesOfType("must_redact"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("must_redact"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("must_redact entry was found.");
@ -252,7 +252,7 @@ rule "11: Redact if CTL/* or BL/* was found"
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");

48
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -78,10 +78,10 @@ rule "0: Expand CBI_author and PII entities with salutation prefix"
rule "1: Redacted because Section contains Vertebrate"
when
$section: Section(hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(1);
@ -92,10 +92,10 @@ rule "1: Redacted because Section contains Vertebrate"
rule "2: Not Redacted because Section contains no Vertebrate"
when
$section: Section(!hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(!hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(2);
@ -105,11 +105,11 @@ rule "2: Not Redacted because Section contains no Vertebrate"
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(3);
@ -119,12 +119,12 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(4);
@ -136,12 +136,12 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
rule "5: Do not redact Names and Addresses if published information found"
when
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("published_information"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
List<RedactionEntity> publishedInformationEntities = $section.getActiveEntitiesOfType("published_information");
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
@ -192,10 +192,10 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate
rule "8: Redact if must_redact entity is found"
when
$section: Section(hasEntitiesOfType("must_redact"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
$section: Section(hasActiveEntitiesOfType("must_redact"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("must_redact entry was found.");
@ -251,7 +251,7 @@ rule "11: Redact if CTL/* or BL/* was found"
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");