From 97d32912fc549ca75f7913f69cdd65a25f2951d0 Mon Sep 17 00:00:00 2001 From: Kilian Schuettler Date: Wed, 5 Jul 2023 22:58:54 +0200 Subject: [PATCH] RED-6929: fix acceptance tests/rules --- .../graph/entity/MatchedRuleHolder.java | 23 ++++++ .../services/EntityCreationService.java | 29 ++++++- .../service/EntityRedactionService.java | 23 ++---- ...ManualRedactionSurroundingTextService.java | 2 +- .../resources/drools/acceptance_rules.drl | 45 ++--------- .../src/test/resources/drools/all_rules.drl | 76 ++----------------- .../test/resources/drools/documine_flora.drl | 2 - .../src/test/resources/drools/rules.drl | 61 ++------------- .../src/test/resources/drools/rules_v2.drl | 6 +- 9 files changed, 80 insertions(+), 187 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java index 3da9d1b8..45ce83db 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java @@ -33,6 +33,29 @@ public interface MatchedRuleHolder { } + default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) { + + getMatchedRuleList().add(MatchedRule.builder() + .ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)) + .reason(reason) + .legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis)) + .applied(true) + .build()); + } + + + private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) { + + if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) { + if (getMatchedRule() == null || !getMatchedRule().isApplied()) { + return "n-a"; + } + return getMatchedRule().getLegalBasis(); + } + return legalBasis; + } + + default void applyWithLineBreaks(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) { if (legalBasis.isBlank() || legalBasis.isEmpty()) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java index 117bb07d..f7b40426 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java @@ -138,6 +138,22 @@ public class EntityCreationService { } + public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation, + String type, + EntityType entityType, + SemanticNode node, + boolean isDossierDictionaryEntry) { + + searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary()) + .stream() + .filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary)) + .map(bounds -> forceByBoundary(bounds, type, entityType, node)) + .peek(entity -> entity.setDictionaryEntry(true)) + .peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry)) + .forEach(entity -> entity.addEngine(Engine.DICTIONARY)); + } + + public Stream lineAfterStrings(List strings, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.getTextBlock(); @@ -304,6 +320,7 @@ public class EntityCreationService { return Optional.empty(); } addEntityToGraph(entity, node); + entity.addEngine(Engine.RULE); return Optional.of(entity); } @@ -343,15 +360,19 @@ public class EntityCreationService { } - public Optional byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { + public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) { - return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny(); + var entity = forceByBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode); + entity.addEngine(Engine.NER); + return entity; } - public Optional byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { + public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) { - return byBoundary(nerEntity.boundary(), type, entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny(); + var entity = forceByBoundary(nerEntity.boundary(), type, entityType, semanticNode); + entity.addEngine(Engine.NER); + return entity; } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java index 2de49c28..2e352ccf 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java @@ -9,9 +9,7 @@ import org.springframework.stereotype.Service; import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest; import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; -import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType; -import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService; @@ -73,21 +71,14 @@ public class EntityRedactionService { public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) { for (var model : dictionary.getDictionaryModels()) { - entityCreationService.bySearchImplementation(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node) - .forEach(entity -> setFields(entity, model.isDossierDictionary())); - entityCreationService.bySearchImplementation(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node) - .forEach(entity -> setFields(entity, model.isDossierDictionary())); - entityCreationService.bySearchImplementation(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node) - .forEach(entity -> setFields(entity, model.isDossierDictionary())); + entityCreationService.bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary()); + entityCreationService.bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary()); + entityCreationService.bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), + model.getType(), + EntityType.FALSE_RECOMMENDATION, + node, + model.isDossierDictionary()); } } - - private void setFields(RedactionEntity entity, boolean isDossierDictionary) { - - entity.setDictionaryEntry(true); - entity.setDossierDictionaryEntry(isDossierDictionary); - entity.addEngine(Engine.DICTIONARY); - } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java index c914b140..210e3a55 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java @@ -77,7 +77,7 @@ public class ManualRedactionSurroundingTextService { .stream() .map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node)) .collect(Collectors.toSet()); - + entities.forEach(RedactionEntity::removeFromGraph); RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions); return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter()); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl index 87452a6d..09c25230 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl @@ -66,7 +66,6 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { entity.skip("SYN.1.0", ""); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -127,8 +126,8 @@ rule "CBI.2.0: Don't redact genitive CBI_author" rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables" when $section: Section(!hasTables(), - hasEntitiesOfType("published_information"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + hasEntitiesOfType("published_information"), + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -143,7 +142,7 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row" when $table: Table(hasEntitiesOfType("published_information"), - (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) + (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(redactionEntity -> { @@ -169,7 +168,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -186,7 +184,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -205,7 +202,6 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -222,7 +218,6 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -249,7 +244,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); insert(entity); }); @@ -264,7 +258,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); @@ -278,7 +271,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); insert(entity); }); @@ -290,7 +282,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); insert(entity); }); @@ -307,7 +298,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -322,7 +312,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -357,7 +346,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -370,7 +358,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -394,7 +381,6 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) .forEach(contactEntity -> { - contactEntity.addEngine(Engine.RULE); contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(contactEntity); }); @@ -416,7 +402,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) .forEach(contactEntity -> { - contactEntity.addEngine(Engine.RULE); contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(contactEntity); }); @@ -432,7 +417,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -445,7 +429,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -458,7 +441,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -471,7 +453,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -485,10 +466,7 @@ rule "ETC.0.0: Purity Hint" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) - .forEach(hint -> { - hint.addEngine(Engine.RULE); - hint.skip("ETC.0.0", ""); - }); + .forEach(hint -> hint.skip("ETC.0.0", "")); end @@ -550,8 +528,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -563,9 +539,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) + .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -624,7 +598,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis); $entityToForce.setRemoved(false); $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); @@ -643,8 +617,8 @@ rule "MAN.3.0: Apply image recategorization" then $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); update($imageToBeRecategorized); - retract($recategorization); update($imageToBeRecategorized.getParent()); + retract($recategorization); end @@ -765,8 +739,5 @@ rule "LDS.0.0: run local dictionary search" DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - insert(entity); - }); + .forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl index fab5247a..6b5e1849 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl @@ -68,7 +68,6 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) ).forEach(entity -> { entity.skip("SYN.0.0", "hint_only"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -83,7 +82,6 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL" entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { entity.skip("SYN.1.0", ""); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -147,7 +145,6 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate" then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.3.0", "Vertebrate found", @@ -164,7 +161,6 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate" $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.3.1", "Vertebrate found", @@ -179,10 +175,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate" $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.3.2", "No vertebrate found"); - }); + .forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found")); end rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" @@ -191,10 +184,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" then $table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.3.3", "No vertebrate found"); - }); + .forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found")); end @@ -208,7 +198,6 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skipWithReferences( "CBI.4.0", "Vertebrate but a no redaction indicator found", @@ -226,7 +215,6 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skipWithReferences( "CBI.4.1", "Vertebrate but a no redaction indicator found", @@ -360,7 +348,6 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity" then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.8.0", "must_redact entity found", @@ -377,7 +364,6 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity" $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.8.1", "must_redact entity found", @@ -401,7 +387,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -418,7 +403,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -437,7 +421,6 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -454,7 +437,6 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -486,7 +468,6 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author" .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.skip("CBI.12.0", "Author(s) header found"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -595,7 +576,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); insert(entity); }); @@ -610,7 +590,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); @@ -624,7 +603,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); insert(entity); }); @@ -636,7 +614,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); insert(entity); }); @@ -688,7 +665,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -703,7 +679,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -738,7 +713,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -751,7 +725,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -775,7 +748,6 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)" then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) .forEach(contactEntity -> { - contactEntity.addEngine(Engine.RULE); contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(contactEntity); }); @@ -797,7 +769,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)" then entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section) .forEach(contactEntity -> { - contactEntity.addEngine(Engine.RULE); contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); insert(contactEntity); }); @@ -813,7 +784,6 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)" entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -826,7 +796,6 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)" entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -860,7 +829,6 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -892,7 +860,6 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -911,7 +878,6 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -928,7 +894,6 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -946,7 +911,6 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)" ) .forEach(contactEntity -> { contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -962,7 +926,6 @@ rule "PII.6.1: redact line between contact keywords" ) .forEach(contactEntity -> { contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -986,7 +949,6 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate )) .forEach(entity -> { entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -1008,7 +970,6 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate )) .forEach(entity -> { entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -1032,7 +993,6 @@ rule "PII.8.0: Redact contact information if producer is found" )) .forEach(entity -> { entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -1054,7 +1014,6 @@ rule "PII.8.1: Redact contact information if producer is found" )) .forEach(entity -> { entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -1069,7 +1028,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -1082,7 +1040,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -1095,7 +1052,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -1108,7 +1064,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -1148,7 +1103,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix" entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - expandedEntity.addEngine(Engine.RULE); insert(expandedEntity); }); end @@ -1162,10 +1116,7 @@ rule "ETC.0.0: Purity Hint" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section) - .forEach(hint -> { - hint.addEngine(Engine.RULE); - hint.skip("ETC.0.0", ""); - }); + .forEach(hint -> hint.skip("ETC.0.0", "")); end @@ -1175,10 +1126,7 @@ rule "ETC.1.0: Redact Purity" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegex("\\bPurity:\\s*(?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); - entity.addEngine(Engine.RULE); - }); + .forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)")); end @@ -1295,8 +1243,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -1308,9 +1254,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) + .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -1327,9 +1271,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author" nerEntities.getNerEntityList().stream() .filter(nerEntity -> !nerEntity.type().equals("CBI_author")) .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) .forEach(entity -> insert(entity)); + end @@ -1384,7 +1327,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis); $entityToForce.setRemoved(false); $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); @@ -1525,8 +1468,5 @@ rule "LDS.0.0: run local dictionary search" DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - insert(entity); - }); + .forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl index 00c9a4d6..342152fc 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl @@ -374,8 +374,6 @@ rule "DOC.8.1: Performing Laboratory (Name)" nerEntities.streamEntitiesOfType("COUNTRY") .filter(nerEntity -> $section.getBoundary().contains(nerEntity.boundary())) .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section)) - .filter(Optional::isPresent) - .map(Optional::get) .forEach(entity -> { entity.apply("DOC.8.2", "Performing Laboratory found", "n-a"); insert(entity); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index ddabbc57..869cbee8 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -68,7 +68,6 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)" entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section) ).forEach(entity -> { entity.skip("SYN.0.0", "hint_only"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -83,7 +82,6 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate" then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.3.0", "Vertebrate found", @@ -100,7 +98,6 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate" $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.3.1", "Vertebrate found", @@ -115,10 +112,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate" $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.3.2", "No vertebrate found"); - }); + .forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found")); end rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" @@ -127,10 +121,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate" then $table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - entity.skip("CBI.3.3", "No vertebrate found"); - }); + .forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found")); end @@ -144,7 +135,6 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skipWithReferences( "CBI.4.0", "Vertebrate but a no redaction indicator found", @@ -162,7 +152,6 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skipWithReferences( "CBI.4.1", "Vertebrate but a no redaction indicator found", @@ -223,7 +212,6 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity" then $section.getEntitiesOfType(List.of("CBI_author", "CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.8.0", "must_redact entity found", @@ -240,7 +228,6 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity" $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact")) .filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address")) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.applyWithReferences( "CBI.8.1", "must_redact entity found", @@ -264,7 +251,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -281,7 +267,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -313,7 +298,6 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author" .map(Optional::get) .forEach(redactionEntity -> { redactionEntity.skip("CBI.12.0", "Author(s) header found"); - redactionEntity.addEngine(Engine.RULE); insert(redactionEntity); }); end @@ -411,7 +395,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); insert(entity); }); @@ -426,7 +409,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)" entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section) .forEach(entity -> { entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false); }); @@ -440,7 +422,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"); insert(entity); }); @@ -452,7 +433,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with then entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section) .forEach(entity -> { - entity.addEngine(Engine.RULE); entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"); insert(entity); }); @@ -504,7 +484,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -519,7 +498,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section) .forEach(laboratoryEntity -> { laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - laboratoryEntity.addEngine(Engine.RULE); dictionary.addLocalDictionaryEntry(laboratoryEntity); insert(laboratoryEntity); }); @@ -554,7 +532,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -567,7 +544,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)" then entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section) .forEach(emailEntity -> { - emailEntity.addEngine(Engine.RULE); emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"); insert(emailEntity); }); @@ -602,7 +578,6 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -634,7 +609,6 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section) .forEach(contactEntity -> { contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -652,7 +626,6 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)" ) .forEach(contactEntity -> { contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -668,7 +641,6 @@ rule "PII.6.1: redact line between contact keywords" ) .forEach(contactEntity -> { contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - contactEntity.addEngine(Engine.RULE); insert(contactEntity); }); end @@ -692,7 +664,6 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate )) .forEach(entity -> { entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -714,7 +685,6 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate )) .forEach(entity -> { entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -738,7 +708,6 @@ rule "PII.8.0: Redact contact information if producer is found" )) .forEach(entity -> { entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -760,7 +729,6 @@ rule "PII.8.1: Redact contact information if producer is found" )) .forEach(entity -> { entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - entity.addEngine(Engine.RULE); insert(entity); }); end @@ -775,7 +743,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -788,7 +755,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -801,7 +767,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -814,7 +779,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section) .forEach(authorEntity -> { authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); - authorEntity.addEngine(Engine.RULE); insert(authorEntity); }); end @@ -841,7 +805,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix" entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*") .ifPresent(expandedEntity -> { expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()); - expandedEntity.addEngine(Engine.RULE); insert(expandedEntity); }); end @@ -855,10 +818,7 @@ rule "ETC.1.0: Redact Purity" $section: Section(containsStringIgnoreCase("purity")) then entityCreationService.byRegex("\\bPurity:\\s*(?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section) - .forEach(entity -> { - entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); - entity.addEngine(Engine.RULE); - }); + .forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)")); end @@ -975,8 +935,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author" then nerEntities.streamEntitiesOfType("CBI_author") .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) .forEach(entity -> insert(entity)); end @@ -988,9 +946,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address" nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY")) then nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities) - .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) - .filter(Optional::isPresent) - .map(Optional::get) + .map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document)) .forEach(entity -> { entity.addEngine(Engine.NER); insert(entity); @@ -1049,7 +1005,7 @@ rule "MAN.2.0: Apply force redaction" $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis) $entityToForce: RedactionEntity(matchesAnnotationId($id)) then - $entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis); + $entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis); $entityToForce.setRemoved(false); $entityToForce.setIgnored(false); $entityToForce.setSkipRemoveEntitiesContainedInLarger(true); @@ -1068,8 +1024,8 @@ rule "MAN.3.0: Apply image recategorization" then $imageToBeRecategorized.setImageType(ImageType.fromString($imageType)); update($imageToBeRecategorized); - retract($recategorization); update($imageToBeRecategorized.getParent()); + retract($recategorization); end @@ -1190,8 +1146,5 @@ rule "LDS.0.0: run local dictionary search" DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) - .forEach(entity -> { - entity.addEngine(Engine.RULE); - insert(entity); - }); + .forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl index 2c1a72c8..746e718f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl @@ -57,10 +57,7 @@ rule "add NER Entities of type CBI_author or CBI_address" $nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address")) then entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document) - .ifPresent(redactionEntity -> { - redactionEntity.addEngine(Engine.NER); - insert(redactionEntity); - }); + .ifPresent(redactionEntity -> insert(redactionEntity)); end // --------------------------------------- CBI rules ------------------------------------------------------------------- @@ -201,7 +198,6 @@ rule "LDS.0.0: run local dictionary search" then entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document) .forEach(entity -> { - entity.addEngine(Engine.RULE); insert(entity); }); end -- 2.47.2