diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java index 10de154a..66268838 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java @@ -96,6 +96,12 @@ public class RedactionEntity { } + public boolean isActive() { + + return !ignored && !removed; + } + + public void addIntersectingNode(SemanticNode containingNode) { intersectingNodes.add(containingNode); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java index 12c15e88..92825150 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java @@ -188,9 +188,9 @@ public interface SemanticNode { * @param type string representing the type of entity to check for * @return true, if this SemanticNode has at least one Entity of the provided type */ - default boolean hasEntitiesOfType(String type) { + default boolean hasActiveEntitiesOfType(String type) { - return getEntities().stream().anyMatch(redactionEntity -> redactionEntity.getType().equals(type)); + return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> redactionEntity.getType().equals(type)); } @@ -200,9 +200,9 @@ public interface SemanticNode { * @param type string representing the type of entities to return * @return List of RedactionEntities of any the type */ - default List getEntitiesOfType(String type) { + default List getActiveEntitiesOfType(String type) { - return getEntities().stream().filter(redactionEntity -> redactionEntity.getType().equals(type)).toList(); + return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList(); } @@ -212,9 +212,9 @@ public interface SemanticNode { * @param types A list of strings representing the types of entities to return * @return List of RedactionEntities of any provided type */ - default List getEntitiesOfType(List types) { + default List getActiveEntitiesOfType(List types) { - return getEntities().stream().filter(redactionEntity -> redactionEntity.isAnyType(types)).toList(); + return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java index b074832d..e2a3c6b7 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java @@ -107,9 +107,10 @@ public class Table implements SemanticNode { * @param types type strings to check whether a row contains an entity like them * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types. */ - public Stream streamEntitiesWhereRowContainsEntitiesOfType(List types) { + public Stream streamEntitiesWhereRowContainsActiveEntitiesOfType(List types) { List rowsWithEntityOfType = getEntities().stream() + .filter(RedactionEntity::isActive) .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType()))) .map(RedactionEntity::getIntersectingNodes) .filter(node -> node instanceof TableCell) @@ -127,11 +128,14 @@ public class Table implements SemanticNode { * @param types type strings to check whether a row contains an entity like them * @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types. */ - public Stream streamEntitiesWhereRowContainsNoEntitiesOfType(List types) { + public Stream streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List types) { return IntStream.range(0, numberOfRows) .boxed() - .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities).flatMap(Collection::stream).noneMatch(entity -> types.contains(entity.getType()))) + .filter(rowNumber -> streamRow(rowNumber).map(TableCell::getEntities) + .flatMap(Collection::stream) + .filter(RedactionEntity::isActive) + .noneMatch(entity -> types.contains(entity.getType()))) .flatMap(this::streamRow) .map(TableCell::getEntities) .flatMap(Collection::stream); @@ -273,14 +277,14 @@ public class Table implements SemanticNode { * @param redactionEntity the entity, which appears in the row to search * @return List of all entities of the provided type, which appear in the same row that the provided entity appears in. */ - public List getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) { + public List getActiveEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) { return redactionEntity.getIntersectingNodes() .stream() .filter(node -> node instanceof TableCell) .map(node -> (TableCell) node) .flatMap(tableCellNode -> streamRow(tableCellNode.getRow())) - .map(cell -> cell.getEntitiesOfType(type)) + .map(cell -> cell.getActiveEntitiesOfType(type)) .flatMap(Collection::stream) .toList(); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 0b3359b9..2668eb8f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -74,24 +74,24 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" // Rule unit: CBI.3 rule "CBI.3.0: Redacted because Section contains Vertebrate" when - $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(!hasTables(), hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { entity.setRedactionReason("Vertebrate found"); entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(3); - entity.addReferences($section.getEntitiesOfType("vertebrate")); + entity.addReferences($section.getActiveEntitiesOfType("vertebrate")); }); end rule "CBI.3.1: Redacted because Table Row contains Vertebrate" when - $table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $table: Table(hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) + $table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { entity.setRedactionReason("Vertebrate found"); @@ -99,15 +99,15 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate" entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(3); - entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity)); }); end rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate" when - $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(!hasTables(), !hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { entity.setRedactionReason("No vertebrate found"); entity.setRedaction(false); @@ -118,9 +118,9 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate" rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" when - $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) + $table: Table(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")) then - $table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate")) + $table.streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { entity.setRedactionReason("No vertebrate found"); @@ -135,35 +135,35 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section" when $section: Section(!hasTables(), - hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("no_redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { entity.setRedactionReason("Vertebrate but a no redaction indicator found"); entity.setRedaction(false); entity.addEngine(Engine.RULE); entity.addMatchedRule(4); - entity.addReferences($section.getEntitiesOfType("no_redaction_indicator")); + entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator")); }); end rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $table: Table(hasActiveEntitiesOfType("no_redaction_indicator"), + hasActiveEntitiesOfType("vertebrate"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) + $table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { entity.setRedactionReason("Vertebrate but a no redaction indicator found"); entity.setRedaction(false); entity.addEngine(Engine.RULE); entity.addMatchedRule(4); - entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity)); - entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity)); }); end @@ -172,29 +172,29 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section" when $section: Section(!hasTables(), - hasEntitiesOfType("redaction_indicator"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasActiveEntitiesOfType("redaction_indicator"), + hasActiveEntitiesOfType("no_redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found"); entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(5); - entity.addReferences($section.getEntitiesOfType("no_redaction_indicator")); - entity.addReferences($section.getEntitiesOfType("redaction_indicator")); + entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator")); + entity.addReferences($section.getActiveEntitiesOfType("redaction_indicator")); }); end rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row" when - $table: Table(hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $table: Table(hasActiveEntitiesOfType("no_redaction_indicator"), + hasActiveEntitiesOfType("redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator")) + $table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found"); @@ -202,8 +202,8 @@ rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also red entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(5); - entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity)); - entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity)); }); end @@ -211,24 +211,24 @@ rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also red // Rule unit: CBI.8 rule "CBI.8.0: Redacted because Section contains must_redact entity" when - $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(!hasTables(), hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { entity.setRedactionReason("must_redact entity found"); entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)"); entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(15); - entity.addReferences($section.getEntitiesOfType("must_redact")); + entity.addReferences($section.getActiveEntitiesOfType("must_redact")); }); end rule "CBI.8.1: Redacted because Table Row contains must_redact entity" when - $table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $table: Table(hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) + $table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("must_redact")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { entity.setRedactionReason("must_redact entity found"); @@ -236,7 +236,7 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity" entity.setRedaction(true); entity.addEngine(Engine.RULE); entity.addMatchedRule(15); - entity.addReferences($table.getEntitiesOfTypeInSameRow("must_redact", entity)); + entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("must_redact", entity)); }); end @@ -290,7 +290,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then - $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + $table.getActiveEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end @@ -375,7 +375,7 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section) .forEach(keywordEntity -> insert(keywordEntity)); - $section.getEntitiesOfType(List.of($keyword, $residueKeyword)) + $section.getActiveEntitiesOfType(List.of($keyword, $residueKeyword)) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.addMatchedRule(15); @@ -507,13 +507,50 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix" end +// Rule unit: CBI.20 +rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + not FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.setRedaction(false); + laboratoryEntity.addMatchedRule(20); + laboratoryEntity.addEngine(Engine.RULE); + laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study"); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + +rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" + when + FileAttribute(label == "Vertebrate Study", value == "Yes") + $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) + then + entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) + .forEach(laboratoryEntity -> { + laboratoryEntity.setRedaction(true); + laboratoryEntity.addMatchedRule(20); + laboratoryEntity.addEngine(Engine.RULE); + laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found"); + laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); + dictionary.addLocalDictionaryEntry(laboratoryEntity); + insert(laboratoryEntity); + }); + end + + //------------------------------------ PII rules ------------------------------------ // Rule unit: PII.0 rule "PII.0.0: Redact all PII (non vertebrate study)" when not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $pii: RedactionEntity(type == "PII", redaction == false) + $pii: RedactionEntity(type == "PII", dictionaryEntry) then $pii.setRedaction(true); $pii.setRedactionReason("Personal Information found"); @@ -524,7 +561,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)" rule "PII.0.1: Redact all PII (vertebrate study)" when FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") - $pii: RedactionEntity(type == "PII", redaction == false) + $pii: RedactionEntity(type == "PII", dictionaryEntry) then $pii.setRedaction(true); $pii.setRedactionReason("Personal Information found"); @@ -596,6 +633,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> { contactEntity.setRedaction(true); contactEntity.addMatchedRule(4); + contactEntity.addEngine(Engine.RULE); contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword"); contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); insert(contactEntity); @@ -630,6 +668,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st .forEach(contactEntity -> { contactEntity.setRedaction(true); contactEntity.addMatchedRule(4); + contactEntity.addEngine(Engine.RULE); contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword"); contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)"); insert(contactEntity); @@ -650,6 +689,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)" .forEach(contactEntity -> { contactEntity.setRedaction(true); contactEntity.addMatchedRule(6); + contactEntity.addEngine(Engine.RULE); contactEntity.setRedactionReason("Found between contact keywords"); contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(contactEntity); @@ -668,6 +708,7 @@ rule "PII.6.1: redact line between contact keywords" .forEach(contactEntity -> { contactEntity.setRedaction(true); contactEntity.addMatchedRule(6); + contactEntity.addEngine(Engine.RULE); contactEntity.setRedactionReason("Found between contact keywords"); contactEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(contactEntity); @@ -845,66 +886,13 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte end -// Rule unit: PII.10 -rule "PII.10.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)" - agenda-group "LOCAL_DICTIONARY_ADDS" - when - not FileAttribute(label == "Vertebrate Study", value == "Yes") - $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) - then - entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) - .forEach(laboratoryEntity -> { - laboratoryEntity.setRedaction(false); - laboratoryEntity.addMatchedRule(10); - laboratoryEntity.addEngine(Engine.RULE); - laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study"); - dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); - }); - end - -rule "PII.10.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)" - agenda-group "LOCAL_DICTIONARY_ADDS" - when - FileAttribute(label == "Vertebrate Study", value == "Yes") - $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:")) - then - entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) - .forEach(laboratoryEntity -> { - laboratoryEntity.setRedaction(true); - laboratoryEntity.addMatchedRule(10); - laboratoryEntity.addEngine(Engine.RULE); - laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found"); - laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002"); - dictionary.addLocalDictionaryEntry(laboratoryEntity); - insert(laboratoryEntity); - }); - end - - // Rule unit: PII.12 -rule "PII.12.0: Redact On behalf of Sequani Ltd.:" - when - $section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) - then - entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) - .forEach(authorEntity -> { - authorEntity.setRedaction(true); - authorEntity.addMatchedRule(12); - authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found"); - authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002"); - insert(authorEntity); - }); - end - - -// Rule unit: PII.13 -rule "PII.13.0: Expand PII entities with salutation prefix" +rule "PII.12.0: Expand PII entities with salutation prefix" when $entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")) then RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"); - expandedEntity.addMatchedRule(13); + expandedEntity.addMatchedRule(12); insert(expandedEntity); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl index 47f93f29..9baf5a54 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl @@ -79,10 +79,10 @@ rule "0: Expand CBI_author and PII entities with salutation prefix" rule "1: Redacted because Section contains Vertebrate" when - $section: Section(hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.addMatchedRule(1); @@ -93,10 +93,10 @@ rule "1: Redacted because Section contains Vertebrate" rule "2: Not Redacted because Section contains no Vertebrate" when - $section: Section(!hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(!hasActiveEntitiesOfType("vertebrate"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.addMatchedRule(2); @@ -106,11 +106,11 @@ rule "2: Not Redacted because Section contains no Vertebrate" rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("no_redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.addMatchedRule(3); @@ -120,12 +120,12 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("no_redaction_indicator"), + hasActiveEntitiesOfType("redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.addMatchedRule(4); @@ -137,12 +137,12 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi rule "5: Do not redact Names and Addresses if published information found" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("published_information"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("published_information"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - List publishedInformationEntities = $section.getEntitiesOfType("published_information"); - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + List publishedInformationEntities = $section.getActiveEntitiesOfType("published_information"); + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section"); @@ -193,10 +193,10 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate rule "8: Redact if must_redact entity is found" when - $section: Section(hasEntitiesOfType("must_redact"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("must_redact"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.setRedactionReason("must_redact entry was found."); @@ -252,7 +252,7 @@ rule "11: Redact if CTL/* or BL/* was found" entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section) .forEach(mustRedactEntity -> insert(mustRedactEntity)); - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.setRedactionReason("Laboratory for vertebrate studies found"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl index f97c680f..aa00f81c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl @@ -78,10 +78,10 @@ rule "0: Expand CBI_author and PII entities with salutation prefix" rule "1: Redacted because Section contains Vertebrate" when - $section: Section(hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.addMatchedRule(1); @@ -92,10 +92,10 @@ rule "1: Redacted because Section contains Vertebrate" rule "2: Not Redacted because Section contains no Vertebrate" when - $section: Section(!hasEntitiesOfType("vertebrate"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(!hasActiveEntitiesOfType("vertebrate"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.addMatchedRule(2); @@ -105,11 +105,11 @@ rule "2: Not Redacted because Section contains no Vertebrate" rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("no_redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.addMatchedRule(3); @@ -119,12 +119,12 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("no_redaction_indicator"), - hasEntitiesOfType("redaction_indicator"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("no_redaction_indicator"), + hasActiveEntitiesOfType("redaction_indicator"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.addMatchedRule(4); @@ -136,12 +136,12 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi rule "5: Do not redact Names and Addresses if published information found" when - $section: Section(hasEntitiesOfType("vertebrate"), - hasEntitiesOfType("published_information"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("vertebrate"), + hasActiveEntitiesOfType("published_information"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - List publishedInformationEntities = $section.getEntitiesOfType("published_information"); - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + List publishedInformationEntities = $section.getActiveEntitiesOfType("published_information"); + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(false); redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section"); @@ -192,10 +192,10 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate rule "8: Redact if must_redact entity is found" when - $section: Section(hasEntitiesOfType("must_redact"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + $section: Section(hasActiveEntitiesOfType("must_redact"), + (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))) then - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.setRedactionReason("must_redact entry was found."); @@ -251,7 +251,7 @@ rule "11: Redact if CTL/* or BL/* was found" entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section) .forEach(mustRedactEntity -> insert(mustRedactEntity)); - $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) + $section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { redactionEntity.setRedaction(true); redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");