Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6929: fix acceptance tests/rules #33

Merged
kilian.schuettler1 merged 1 commits from RED-6929 into master 2023-07-05 23:06:04 +02:00
Conversation 2 Commits 1 Files Changed 9 +80 -187
9 changed files with 80 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java
View File

@ -33,6 +33,29 @@ public interface MatchedRuleHolder {
}
default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) {
getMatchedRuleList().add(MatchedRule.builder()
.ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
.reason(reason)
.legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis))
.applied(true)
.build());
}
private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) {
if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) {
if (getMatchedRule() == null || !getMatchedRule().isApplied()) {
return "n-a";
}
return getMatchedRule().getLegalBasis();
}
return legalBasis;
}
default void applyWithLineBreaks(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
if (legalBasis.isBlank() || legalBasis.isEmpty()) {

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -138,6 +138,22 @@ public class EntityCreationService {
}
public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation,
String type,
EntityType entityType,
SemanticNode node,
boolean isDossierDictionaryEntry) {
searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
.map(bounds -> forceByBoundary(bounds, type, entityType, node))
.peek(entity -> entity.setDictionaryEntry(true))
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
.forEach(entity -> entity.addEngine(Engine.DICTIONARY));
}
public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
@ -304,6 +320,7 @@ public class EntityCreationService {
return Optional.empty();
}
addEntityToGraph(entity, node);
entity.addEngine(Engine.RULE);
return Optional.of(entity);
}
@ -343,15 +360,19 @@ public class EntityCreationService {
}
public Optional<RedactionEntity> byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, EntityType entityType, SemanticNode semanticNode) {
return byBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny();
var entity = forceByBoundary(nerEntity.boundary(), nerEntity.type(), entityType, semanticNode);
entity.addEngine(Engine.NER);
return entity;
}
public Optional<RedactionEntity> byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
public RedactionEntity byNerEntity(NerEntities.NerEntity nerEntity, String type, EntityType entityType, SemanticNode semanticNode) {
return byBoundary(nerEntity.boundary(), type, entityType, semanticNode).stream().peek(entity -> entity.addEngine(Engine.NER)).findAny();
var entity = forceByBoundary(nerEntity.boundary(), type, entityType, semanticNode);
entity.addEngine(Engine.NER);
return entity;
}

23
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -9,9 +9,7 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
@ -73,21 +71,14 @@ public class EntityRedactionService {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) {
entityCreationService.bySearchImplementation(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node)
.forEach(entity -> setFields(entity, model.isDossierDictionary()));
entityCreationService.bySearchImplementation(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node)
.forEach(entity -> setFields(entity, model.isDossierDictionary()));
entityCreationService.bySearchImplementation(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node)
.forEach(entity -> setFields(entity, model.isDossierDictionary()));
entityCreationService.bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
entityCreationService.bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
entityCreationService.bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
model.getType(),
EntityType.FALSE_RECOMMENDATION,
node,
model.isDossierDictionary());
}
}
private void setFields(RedactionEntity entity, boolean isDossierDictionary) {
entity.setDictionaryEntry(true);
entity.setDossierDictionaryEntry(isDossierDictionary);
entity.addEngine(Engine.DICTIONARY);
}
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -77,7 +77,7 @@ public class ManualRedactionSurroundingTextService {
.stream()
.map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))
.collect(Collectors.toSet());
entities.forEach(RedactionEntity::removeFromGraph);
RedactionEntity correctEntity = getEntityOnCorrectPosition(entities, toFindPositions);
return Pair.of(correctEntity.getTextBefore(), correctEntity.getTextAfter());
}

45
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -66,7 +66,6 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("SYN.1.0", "");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -127,8 +126,8 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
when
$section: Section(!hasTables(),
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
@ -143,7 +142,7 @@ rule "CBI.7.0: Do not redact Names and Addresses if published information found
rule "CBI.7.1: Do not redact Names and Addresses if published information found in same table row"
when
$table: Table(hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
@ -169,7 +168,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -186,7 +184,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -205,7 +202,6 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -222,7 +218,6 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -249,7 +244,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
@ -264,7 +258,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
@ -278,7 +271,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
@ -290,7 +282,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
@ -307,7 +298,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -322,7 +312,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -357,7 +346,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -370,7 +358,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -394,7 +381,6 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.addEngine(Engine.RULE);
contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
@ -416,7 +402,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.addEngine(Engine.RULE);
contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
@ -432,7 +417,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -445,7 +429,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -458,7 +441,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -471,7 +453,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -485,10 +466,7 @@ rule "ETC.0.0: Purity Hint"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
.forEach(hint -> {
hint.addEngine(Engine.RULE);
hint.skip("ETC.0.0", "");
});
.forEach(hint -> hint.skip("ETC.0.0", ""));
end
@ -550,8 +528,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(entity -> insert(entity));
end
@ -563,9 +539,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
@ -624,7 +598,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.setRemoved(false);
$entityToForce.setIgnored(false);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
@ -643,8 +617,8 @@ rule "MAN.3.0: Apply image recategorization"
then
$imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
update($imageToBeRecategorized);
retract($recategorization);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
@ -765,8 +739,5 @@ rule "LDS.0.0: run local dictionary search"
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
.forEach(entity -> insert(entity));
end

76
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -68,7 +68,6 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
).forEach(entity -> {
entity.skip("SYN.0.0", "hint_only");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -83,7 +82,6 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("SYN.1.0", "");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -147,7 +145,6 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate"
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.3.0",
"Vertebrate found",
@ -164,7 +161,6 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.3.1",
"Vertebrate found",
@ -179,10 +175,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.3.2", "No vertebrate found");
});
.forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
end
rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
@ -191,10 +184,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
then
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.3.3", "No vertebrate found");
});
.forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found"));
end
@ -208,7 +198,6 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skipWithReferences(
"CBI.4.0",
"Vertebrate but a no redaction indicator found",
@ -226,7 +215,6 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skipWithReferences(
"CBI.4.1",
"Vertebrate but a no redaction indicator found",
@ -360,7 +348,6 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.8.0",
"must_redact entity found",
@ -377,7 +364,6 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.8.1",
"must_redact entity found",
@ -401,7 +387,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -418,7 +403,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -437,7 +421,6 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -454,7 +437,6 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -486,7 +468,6 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.skip("CBI.12.0", "Author(s) header found");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -595,7 +576,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
@ -610,7 +590,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
@ -624,7 +603,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
@ -636,7 +614,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
@ -688,7 +665,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -703,7 +679,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -738,7 +713,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -751,7 +725,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -775,7 +748,6 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.addEngine(Engine.RULE);
contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
@ -797,7 +769,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.addEngine(Engine.RULE);
contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
@ -813,7 +784,6 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -826,7 +796,6 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -860,7 +829,6 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -892,7 +860,6 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -911,7 +878,6 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -928,7 +894,6 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -946,7 +911,6 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -962,7 +926,6 @@ rule "PII.6.1: redact line between contact keywords"
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -986,7 +949,6 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
))
.forEach(entity -> {
entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -1008,7 +970,6 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
))
.forEach(entity -> {
entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -1032,7 +993,6 @@ rule "PII.8.0: Redact contact information if producer is found"
))
.forEach(entity -> {
entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -1054,7 +1014,6 @@ rule "PII.8.1: Redact contact information if producer is found"
))
.forEach(entity -> {
entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -1069,7 +1028,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -1082,7 +1040,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -1095,7 +1052,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -1108,7 +1064,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -1148,7 +1103,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
expandedEntity.addEngine(Engine.RULE);
insert(expandedEntity);
});
end
@ -1162,10 +1116,7 @@ rule "ETC.0.0: Purity Hint"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
.forEach(hint -> {
hint.addEngine(Engine.RULE);
hint.skip("ETC.0.0", "");
});
.forEach(hint -> hint.skip("ETC.0.0", ""));
end
@ -1175,10 +1126,7 @@ rule "ETC.1.0: Redact Purity"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
entity.addEngine(Engine.RULE);
});
.forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
end
@ -1295,8 +1243,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(entity -> insert(entity));
end
@ -1308,9 +1254,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
@ -1327,9 +1271,8 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author"
nerEntities.getNerEntityList().stream()
.filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(entity -> insert(entity));
end
@ -1384,7 +1327,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.setRemoved(false);
$entityToForce.setIgnored(false);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
@ -1525,8 +1468,5 @@ rule "LDS.0.0: run local dictionary search"
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
.forEach(entity -> insert(entity));
end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -374,8 +374,6 @@ rule "DOC.8.1: Performing Laboratory (Name)"
nerEntities.streamEntitiesOfType("COUNTRY")
.filter(nerEntity -> $section.getBoundary().contains(nerEntity.boundary()))
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(entity -> {
entity.apply("DOC.8.2", "Performing Laboratory found", "n-a");
insert(entity);

61
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -68,7 +68,6 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
).forEach(entity -> {
entity.skip("SYN.0.0", "hint_only");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -83,7 +82,6 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate"
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.3.0",
"Vertebrate found",
@ -100,7 +98,6 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.3.1",
"Vertebrate found",
@ -115,10 +112,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.3.2", "No vertebrate found");
});
.forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
end
rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
@ -127,10 +121,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
then
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.3.3", "No vertebrate found");
});
.forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found"));
end
@ -144,7 +135,6 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skipWithReferences(
"CBI.4.0",
"Vertebrate but a no redaction indicator found",
@ -162,7 +152,6 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skipWithReferences(
"CBI.4.1",
"Vertebrate but a no redaction indicator found",
@ -223,7 +212,6 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.8.0",
"must_redact entity found",
@ -240,7 +228,6 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.applyWithReferences(
"CBI.8.1",
"must_redact entity found",
@ -264,7 +251,6 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -281,7 +267,6 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -313,7 +298,6 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.skip("CBI.12.0", "Author(s) header found");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
});
end
@ -411,7 +395,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
@ -426,7 +409,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
@ -440,7 +422,6 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
@ -452,7 +433,6 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
@ -504,7 +484,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -519,7 +498,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
laboratoryEntity.addEngine(Engine.RULE);
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
@ -554,7 +532,6 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -567,7 +544,6 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.addEngine(Engine.RULE);
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
@ -602,7 +578,6 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -634,7 +609,6 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -652,7 +626,6 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -668,7 +641,6 @@ rule "PII.6.1: redact line between contact keywords"
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002");
contactEntity.addEngine(Engine.RULE);
insert(contactEntity);
});
end
@ -692,7 +664,6 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
))
.forEach(entity -> {
entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -714,7 +685,6 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
))
.forEach(entity -> {
entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -738,7 +708,6 @@ rule "PII.8.0: Redact contact information if producer is found"
))
.forEach(entity -> {
entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -760,7 +729,6 @@ rule "PII.8.1: Redact contact information if producer is found"
))
.forEach(entity -> {
entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addEngine(Engine.RULE);
insert(entity);
});
end
@ -775,7 +743,6 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -788,7 +755,6 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -801,7 +767,6 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -814,7 +779,6 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
authorEntity.addEngine(Engine.RULE);
insert(authorEntity);
});
end
@ -841,7 +805,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
expandedEntity.addEngine(Engine.RULE);
insert(expandedEntity);
});
end
@ -855,10 +818,7 @@ rule "ETC.1.0: Redact Purity"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
entity.addEngine(Engine.RULE);
});
.forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
end
@ -975,8 +935,6 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(entity -> insert(entity));
end
@ -988,9 +946,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.filter(Optional::isPresent)
.map(Optional::get)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
@ -1049,7 +1005,7 @@ rule "MAN.2.0: Apply force redaction"
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.setRemoved(false);
$entityToForce.setIgnored(false);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
@ -1068,8 +1024,8 @@ rule "MAN.3.0: Apply image recategorization"
then
$imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
update($imageToBeRecategorized);
retract($recategorization);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
@ -1190,8 +1146,5 @@ rule "LDS.0.0: run local dictionary search"
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
.forEach(entity -> insert(entity));
end

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -57,10 +57,7 @@ rule "add NER Entities of type CBI_author or CBI_address"
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
then
entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document)
.ifPresent(redactionEntity -> {
redactionEntity.addEngine(Engine.NER);
insert(redactionEntity);
});
.ifPresent(redactionEntity -> insert(redactionEntity));
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
@ -201,7 +198,6 @@ rule "LDS.0.0: run local dictionary search"
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
end