diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 861cc64b..cd6c1888 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -970,7 +970,11 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest { redactionLog.getEntityLogEntry().forEach(entry -> { if (!entry.getEntryType().equals(EntryType.HINT)) { - assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); + if (entry.getType().equals("CBI_author")) { + assertThat(entry.getReason()).isEqualTo("Not redacted because it's row does not belong to a vertebrate study"); + } else if (entry.getType().equals("CBI_address")) { + assertThat(entry.getReason()).isEqualTo("No vertebrate found"); + } } }); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 9c767083..0e08500d 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -218,7 +218,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then - $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl index 0dd7be25..9fb377e1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl @@ -434,42 +434,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then - $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end // Rule unit: CBI.12 -rule "CBI.12.0: Add all cells with Header Author(s) as CBI_author" - salience 1 +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasHeader("Author(s)") || hasHeader("Author")) + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - Stream.concat( - $table.streamTableCellsWithHeader("Author(s)"), - $table.streamTableCellsWithHeader("Author") - ) - .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.1: Do not redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No")) + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" +rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)")); + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 35ed717c..ddf3f3c8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -281,42 +281,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then - $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end // Rule unit: CBI.12 -rule "CBI.12.0: Add all cells with Header Author(s) as CBI_author" - salience 1 +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasHeader("Author(s)") || hasHeader("Author")) + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - Stream.concat( - $table.streamTableCellsWithHeader("Author(s)"), - $table.streamTableCellsWithHeader("Author") - ) - .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.1: Do not redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No")) + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" +rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)")); + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); end diff --git a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl index eed03ce5..27bda14d 100644 --- a/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl +++ b/redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl @@ -434,42 +434,59 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study when $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N")) then - $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); + $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); end // Rule unit: CBI.12 -rule "CBI.12.0: Add all cells with Header Author(s) as CBI_author" - salience 1 +rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasHeader("Author(s)") || hasHeader("Author")) + not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - Stream.concat( - $table.streamTableCellsWithHeader("Author(s)"), - $table.streamTableCellsWithHeader("Author") - ) - .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY)) - .filter(Optional::isPresent) - .map(Optional::get) - .forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.0", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.1: Do not redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No" +rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)" + agenda-group "LOCAL_DICTIONARY_ADDS" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No")) + FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes") + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study")); + + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> { + authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002"); + dictionary.addMultipleAuthorsAsRecommendation(authorEntity); + }); + end -rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes" +rule "CBI.12.2: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'" when - $table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes")) + $table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N")) + TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList() + TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList() + TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList() + $authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList() then - $table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes")) - .filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address"))) - .forEach(authorEntity -> authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)")); + entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY) + .ifPresent(authorEntity -> authorEntity.skip("CBI.12.2", "Not redacted because it's row does not belong to a vertebrate study")); end