Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-7631' into 'master'

Browse Source 
RED-7631: fix recommandations being applied instead of skipped

Closes RED-7631

See merge request redactmanager/redaction-service!195
This commit is contained in:
Kilian Schüttler 2023-11-14 11:37:30 +01:00
commit b499e10590
35 changed files with 890 additions and 4425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -51,6 +51,7 @@ dependencies {
implementation("org.springframework.cloud:spring-cloud-starter-openfeign:4.0.4")
implementation("org.springframework.boot:spring-boot-starter-amqp:3.1.4")
testImplementation(project(":rules-management"))
testImplementation("org.apache.pdfbox:pdfbox:${pdfBoxVersion}")
testImplementation("org.apache.pdfbox:pdfbox-tools:${pdfBoxVersion}")

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/document/entity/MatchedRule.java
View File

@ -31,11 +31,11 @@ public final class MatchedRule implements Comparable<MatchedRule> {
String reason = "";
@Builder.Default
String legalBasis = "";
boolean applied;
boolean writeValueWithLineBreaks;
boolean applied;
boolean removed;
boolean ignored;
boolean resized;
@Builder.Default
Set<TextEntity> references = Collections.emptySet();
@ -46,6 +46,20 @@ public final class MatchedRule implements Comparable<MatchedRule> {
}
public MatchedRule asSkippedIfApplied() {
if (!this.isApplied()) {
return this;
}
return MatchedRule.builder().ruleIdentifier(getRuleIdentifier())
.writeValueWithLineBreaks(this.isWriteValueWithLineBreaks())
.legalBasis(this.getLegalBasis())
.reason(this.getReason())
.references(this.getReferences())
.build();
}
@Override
public int compareTo(MatchedRule matchedRule) {

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/model/drools/BasicRule.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.model.drools;
import org.drools.drl.ast.descr.AttributeDescr;
import org.drools.drl.ast.descr.RuleDescr;
import lombok.AccessLevel;
@ -18,6 +19,7 @@ public class BasicRule {
RuleIdentifier identifier;
String name;
String code;
String agendaGroup;
int line;
@ -26,7 +28,8 @@ public class BasicRule {
RuleIdentifier identifier = RuleIdentifier.fromName(rule.getName());
String nameWithoutIdentifier = rule.getName().replace(identifier + ":", "");
String code = rulesString.substring(rule.getStartCharacter(), rule.getEndCharacter());
return new BasicRule(identifier, nameWithoutIdentifier, code, rule.getLine());
String agendaGroup = rule.getAttributes().getOrDefault("agenda-group", new AttributeDescr("agenda-group", "DEFAULT")).getValue();
return new BasicRule(identifier, nameWithoutIdentifier, code, agendaGroup, rule.getLine());
}

30
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/drools/DroolsSyntaxValidationService.java
View File

@ -5,6 +5,7 @@ import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
import org.drools.drl.parser.DroolsParserException;
import org.kie.api.builder.KieBuilder;
import org.kie.api.builder.Message;
import org.springframework.stereotype.Service;
@ -32,14 +33,20 @@ public class DroolsSyntaxValidationService {
@SneakyThrows
public DroolsSyntaxValidation testRules(RuleValidationModel rules) {
DroolsSyntaxValidation customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
DroolsSyntaxValidation customDroolsSyntaxValidation;
try {
customDroolsSyntaxValidation = buildCustomDroolsSyntaxValidation(rules.getRulesString(), RuleFileType.valueOf(rules.getRuleFileType()));
} catch (DroolsParserException e) {
// this means the parser could not parse the file at all. In this case use drools compiler only as it will return useful error messages.
customDroolsSyntaxValidation = new DroolsSyntaxValidation();
}
DroolsSyntaxValidation droolsCompilerSyntaxValidation = buildDroolsCompilerSyntaxValidation(rules);
droolsCompilerSyntaxValidation.getDroolsSyntaxErrorMessages().addAll(customDroolsSyntaxValidation.getDroolsSyntaxErrorMessages());
return droolsCompilerSyntaxValidation;
}
private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) {
private DroolsSyntaxValidation buildCustomDroolsSyntaxValidation(String ruleString, RuleFileType ruleFileType) throws DroolsParserException {
RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(ruleString);
DroolsSyntaxValidation customSyntaxValidation = ruleFileBluePrint.getDroolsSyntaxValidation();
@ -75,19 +82,24 @@ public class DroolsSyntaxValidationService {
.build());
}
});
baseRuleFileBluePrint.streamAllRules().forEach(basicRule -> {
if (!validateRuleIsPresent(basicRule, ruleFileBluePrint)) {
int line = ruleFileBluePrint.findRulesByIdentifier(basicRule.getIdentifier()).stream().findFirst().map(BasicRule::getLine).orElse(basicRule.getLine());
customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(line)
.column(0)
.message(String.format("Changing or removing the rule %s is not allowed! Must be: %n%s", basicRule.getName(), basicRule.getCode()))
if (ruleFileType.equals(RuleFileType.ENTITY)) {
String requiredAgendaGroup = "LOCAL_DICTIONARY_ADDS";
if (!validateAgendaGroupIsPresent(ruleFileBluePrint, requiredAgendaGroup)) {
customSyntaxValidation.getDroolsSyntaxErrorMessages().add(DroolsSyntaxErrorMessage.builder().line(0)
.column(0).message(String.format("At least one rule with Agenda-Group '%s' required!", requiredAgendaGroup))
.build());
}
});
}
return customSyntaxValidation;
}
private boolean validateAgendaGroupIsPresent(RuleFileBluePrint ruleFileBluePrint, String agendaGroupName) {
return ruleFileBluePrint.streamAllRules().anyMatch(basicRule -> basicRule.getAgendaGroup().equals(agendaGroupName));
}
private boolean importsAreValid(RuleFileBluePrint baseRuleFileBluePrint, RuleFileBluePrint ruleFileBluePrint) {
// imports may shrink, but not add anything new!

19
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/MigrationIntegrationTest.java
View File

@ -4,8 +4,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@ -25,6 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.migration.MigratedIds;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.dossier.file.FileType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
@ -51,6 +54,22 @@ public class MigrationIntegrationTest extends BuildDocumentIntegrationTest {
ObjectMapper mapper;
@Test
@SneakyThrows
public void testSave() {
MigratedIds ids = new MigratedIds(new LinkedList<>());
ids.addMapping("123", "321");
ids.addMapping("123", "321");
ids.addMapping("123", "321");
ids.addMapping("123", "321");
ids.addMapping("123", "321");
mapper.writeValue(new FileOutputStream("/tmp/testIds.json"), ids);
var ids2 = mapper.readValue(new FileInputStream("/tmp/testIds.json"), MigratedIds.class);
assert ids2.getMappings().size() == 5;
}
@Test
@SneakyThrows
public void testMigration() {

53
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/drools/files/management/services/DroolsUpToDateTest.java Normal file
View File

@ -0,0 +1,53 @@
package com.iqser.red.service.redaction.v1.server.drools.files.management.services;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.wildfly.common.Assert.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.springframework.core.io.ClassPathResource;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
import com.knecon.fforesight.utility.rules.management.models.BasicRule;
import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DroolsUpToDateTest {
@Test
@SneakyThrows
public void assertAllRuleFilesAreUpToDate() {
Path droolsPath = new ClassPathResource("drools").getFile().toPath();
Files.walk(droolsPath).filter(DroolsUpToDateTest::isEntityRuleFile).forEach(this::validateFile);
}
private static boolean isEntityRuleFile(Path droolsFile) {
String fileName = droolsFile.getFileName().toString();
return fileName.endsWith(".drl") && !fileName.endsWith("_components.drl") && !fileName.endsWith("_OLD.drl");
}
private void validateFile(Path path) {
log.info(path.toFile().getAbsolutePath());
RuleFileBluePrint allRules = RuleFileParser.buildBluePrintFromAllRuleFiles();
RuleFileBluePrint thisRules = RuleFileParser.buildBluePrintFromRulesString(RuleFileIO.getRulesString(path.toFile().getAbsolutePath()));
assertTrue(allRules.getAllRuleIdentifiers().containsAll(thisRules.getAllRuleIdentifiers()));
for (BasicRule rule : thisRules.getAllRules()) {
List<BasicRule> updatedRule = allRules.findRuleByIdentifier(rule.identifier());
assert updatedRule.size() == 1;
assertEquals(updatedRule.get(0), rule);
}
}
}

136
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -82,25 +82,25 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Redact CBI Authors (Non Vertebrate Study)"
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
then
$entity.apply("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "CBI.0.1: Redact CBI Authors (Vertebrate Study)"
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
then
$entity.apply("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: CBI.1
rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
@ -108,19 +108,19 @@ rule "CBI.1.0: Don't redact CBI Address (Non Vertebrate Study)"
$entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
end
rule "CBI.1.1: Redact CBI Address (Vertebrate Study)"
rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_address", dictionaryEntry)
then
$entity.apply("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.redact("CBI.1.1", "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: CBI.2
rule "CBI.2.0: Don't redact genitive CBI_author"
rule "CBI.2.0: Do not redact genitive CBI Author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -128,7 +128,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
// Rule unit: CBI.7
rule "CBI.7.0: Do not redact Names and Addresses if published information found in section without tables"
rule "CBI.7.0: Do not redact Names and Addresses if published information found in Section without tables"
when
$section: Section(!hasTables(),
hasEntitiesOfType("published_information"),
@ -156,7 +156,7 @@ rule "CBI.7.1: Do not redact Names and Addresses if published information found
// Rule unit: CBI.9
rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -166,10 +166,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -179,12 +179,12 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: CBI.10
rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrate study)"
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -194,10 +194,10 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -207,7 +207,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -223,7 +223,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
// Rule unit: CBI.16
rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -231,12 +231,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -244,7 +244,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
@ -290,7 +290,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(laboratoryEntity);
});
end
@ -304,7 +304,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
then
$pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "PII.0.1: Redact all PII (vertebrate study)"
@ -312,7 +312,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
then
$pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -323,7 +323,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@ -332,7 +332,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -352,7 +352,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
@ -370,45 +370,27 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -430,25 +412,33 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.2.1: Redact signatures (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: ETC.3
rule "ETC.3.0: Redact logos (vertebrate study)"
rule "ETC.3.0: Skip logos (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.skip("ETC.3.0", "Logo Found");
end
rule "ETC.3.1: Redact logos (non vertebrate study)"
rule "ETC.3.1: Redact logos (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -466,7 +456,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
rule "AI.0.0: add all NER Entities of type CBI_author"
rule "AI.0.0: Add all NER Entities of type CBI_author"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@ -477,7 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
// Rule unit: AI.1
rule "AI.1.0: combine and add NER Entities as CBI_address"
rule "AI.1.0: Combine and add NER Entities as CBI_address"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@ -486,7 +476,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -631,7 +621,7 @@ rule "MAN.4.1: Apply legal basis change"
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
@ -643,7 +633,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
// Rule unit: X.1
rule "X.1.0: merge intersecting Entities of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@ -659,7 +649,7 @@ rule "X.1.0: merge intersecting Entities of same type"
// Rule unit: X.2
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@ -672,7 +662,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
// Rule unit: X.3
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@ -684,7 +674,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
// Rule unit: X.4
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -697,7 +687,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
// Rule unit: X.5
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -709,7 +699,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -724,7 +714,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -735,7 +725,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: remove duplicate FileAttributes"
rule "FA.1.0: Remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
@ -757,6 +747,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

916
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/adama-pilot.drl
View File

@ -1,916 +0,0 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//---------------------------------------------------------------------------
rule "H.0.0 retract table of contents page"
when
$page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES")))
$node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)
then
retract($node);
end
rule "H.0.0: Ignore Table of Contents"
salience 10
when
$tocHeadline: Headline(containsString("CONTENTS"))
$page: Page() from $tocHeadline.getParent().getPages()
$node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))
then
retract($node);
end
//---------------------- CUSTOMER RULES -----------------------------------------------------
rule "DOC.1.0: Adama number"
when
$paragraph: Paragraph(
containsString("R-")
&& onPage(1)
)
then
entityCreationService
.byRegex(
"0?0?R\\-[l\\d]{3,5}\\p{Lu}?",
"adama_number",
EntityType.ENTITY,
$paragraph)
.findFirst()
.ifPresent(entity -> {
entity.apply("DOC.1.0", "Adama number found");
insert(FileAttribute.builder().label("Adama").value(entity.getValue()).build());
});
end
rule "DOC.2.0: Study number by keyword"
when
$studyNumberKeyword: String() from List.of(
"BioChem project number:",
"DTI Report.",
"Final Report N°",
"LPT Report No.",
"Project identity",
"Project No.:",
"Protocol No.",
"PTRL Report No.",
"SLI Report #",
"SLI Study #",
"Specht Analytical Study Plan",
"SPL PROJECT NUMBER:",
"Study code:",
"Study N°",
"Study-No.",
"Study No:",
"Study number:",
"Study Number"
)
$excludeKeyWords: String() from List.of(
"Sponsors study number"
)
$paragraph: Paragraph(
containsStringIgnoreCase($studyNumberKeyword)
&& !containsStringIgnoreCase($excludeKeyWords)
&& onPage(1)
)
then
entityCreationService
.lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$paragraph)
.findFirst()
.ifPresent(entity ->
{
entity.apply("DOC.2.0", "Study number found");
insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
}
);
end
rule "DOC.2.1: Study number in header"
when
$studyNumberKeyword: String() from List.of(
"Final Report"
)
$header: Header(
containsStringIgnoreCase($studyNumberKeyword)
&& onPage(2)
)
then
entityCreationService
.lineAfterStringIgnoreCase($studyNumberKeyword,"study_number",EntityType.ENTITY,$header)
.findFirst()
.ifPresent(entity ->
{
entity.apply("DOC.2.1", "Study number found");
insert(FileAttribute.builder().label("Study").value(entity.getValue()).build());
}
);
end
rule "DOC.2.2: Project Number with Section Title"
when
$title: String() from List.of(
"LABORATORY PROJECT ID."
)
$paragraph: Paragraph(
containsStringIgnoreCase($title)
&& onPage(1)
)
then
entityCreationService.semanticNodeAfterString(
$paragraph,
"LABORATORY PROJECT ID.",
"study_number",
EntityType.ENTITY
)
.ifPresent(
entity -> entity.apply("DOC.2.2","Study number by title found.")
);
end
rule "DOC.3.0: Batch Material Number"
when
$hlKeyword: String() from List.of(
"formulation",
"formulation:",
"test item",
"test substance",
"test material"
)
$hlNoKeyword: String() from List.of(
"Preparation"
)
$headline: Headline(
containsStringIgnoreCase($hlKeyword)
&& !containsStringIgnoreCase($hlNoKeyword)
)
$batchKeyword: String() from List.of(
"Batch number",
"Batch number:",
"Batch number* :",
"batch no",
"Batch no.",
"Lot.n.:"
)
$section: Section(
containsStringIgnoreCase($batchKeyword)
&& (
getHeadline() == $headline
|| (
containsStringIgnoreCase($hlKeyword)
&& !containsStringIgnoreCase($hlNoKeyword)
)
)
)
then
entityCreationService
.lineAfterStringIgnoreCase($batchKeyword,"batch_material_number",EntityType.ENTITY,$section)
.forEach(
entity -> {
entity.apply("DOC.3.0","Batch number found.");
insert(FileAttribute.builder().label("Batch No").value(entity.getValue()).build());
}
);
end
rule "DOC.4.0: study title as headline"
when
$hlKeyword: String() from List.of(
"Analyitical Method",
"Residues",
"Acute",
"Process",
"Application",
"Review",
"Examination",
"Toxicity",
"Method Validation",
"Explosion",
"Sensitisation",
"Determination",
"Test"
)
$hlNotKeyword: String() from List.of(
"Laboratoire",
"Laboratorien"
)
$species: TextEntity(type=="species")
$headline: Headline(
onPage(1)
&& (
containsStringIgnoreCase($hlKeyword)
|| entities contains $species
)
&& !containsStringIgnoreCase($hlNotKeyword)
)
then
entityCreationService.bySemanticNode($headline, "title", EntityType.ENTITY).ifPresent(entity -> {
entity.apply("DOC.4.0", "Title found", "n-a");
insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
});
end
rule "DOC.4.1: study title on cover page between sections"
when
not TextEntity(type=="title", applied())
$page: Page(
getNumber() == 1,
getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Title".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Title".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Report 92 50 12 136".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Study Report".toLowerCase()),
getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Guideline".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Identification".toLowerCase())
// too many false positives due to term in header and cover page || getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Final Report".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Data Requirement".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Submitted".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Guideline".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Study Director".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Author".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Including:".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Laboratory Investigations".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Test Article".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "HLS".toLowerCase())
|| getMainBodyTextBlock().getSearchText().toLowerCase() (contains "Official Journal".toLowerCase())
)
then
List<String> startStrings = List.of("Study Title", "Study Title:", "Title", "Final Report", "Final Study Report", "Report 92 50 12 136");
List<String> stopStrings = List.of("Guideline", "Guidelines", "Study Identification", "Data Requirement", "Submitted", "Test Guideline",
"Study Director", "Author", "Including:", "Laboratory Investigations", "Test Article", "HLS", "Official Journal");
// too many false positives due to term in header and cover page stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Final Report", $page.getMainBodyTextBlock()));
entityCreationService.shortestBetweenAnyString(startStrings, stopStrings, "title", EntityType.ENTITY, document).forEach(entity -> {
entity.apply("DOC.4.1", "Study title found", "n-a");
insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
});
end
rule "DOC.4.2: Study title by paragraph including species on cover page"
when
not TextEntity(type=="title", applied())
$species: TextEntity(type=="species")
$paragraph: Paragraph(
onPage(1)
&& entities contains $species
)
then
entityCreationService.bySemanticNode(
$paragraph,
"title",
EntityType.ENTITY
)
.ifPresent(entity -> {
entity.apply("DOC.4.2","Title with species found on cover page");
insert(FileAttribute.builder().label("Title").value(entity.getValue()).build());
});
end
rule "DOC.5.0: Study Director on cover page in a paragraph on line"
when
not TextEntity(type=="author", applied())
$sectionTitle: String() from List.of(
"Study Director/Analyst:",
"Study Director:",
"Study Director :",
"Author:",
"Author "
)
$paragraph: Paragraph(
onPage(1)
&& containsStringIgnoreCase($sectionTitle)
)
then
entityCreationService
.lineAfterStringIgnoreCase(
$sectionTitle,
"author",
EntityType.ENTITY,
$paragraph
)
.forEach(entity -> {
entity.apply("DOC.5.0","Study Director on cover page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.1: Author/Study Director listing on cover page"
when
not TextEntity(type=="author", applied())
$page: Page(
getNumber() == 1
&& getMainBodyTextBlock().getSearchText().toLowerCase() (
contains "Author(s)".toLowerCase()
|| contains "Authors".toLowerCase()
|| contains "Author".toLowerCase()
|| contains "Study Director".toLowerCase()
)
&& getMainBodyTextBlock().getSearchText().toLowerCase() (
contains "Test Facility".toLowerCase()
|| contains "Testing Facility".toLowerCase()
|| contains "Study Initiation".toLowerCase()
|| contains "Study Initiated".toLowerCase()
|| contains "Study completed on".toLowerCase()
|| contains "Study completion".toLowerCase()
|| contains "Study amended".toLowerCase()
|| contains "Report Issue Date".toLowerCase()
|| contains "Document Date".toLowerCase()
|| contains "Date".toLowerCase()
|| contains "Defitrace".toLowerCase() // this should be solved with Organisation Entity as a TextRange
)
)
then
List<TextRange> startBoundaries = new LinkedList<>();
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author(s)", $page.getMainBodyTextBlock()));
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Authors", $page.getMainBodyTextBlock()));
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Author", $page.getMainBodyTextBlock()));
startBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Director", $page.getMainBodyTextBlock()));
List<TextRange> stopBoundaries = new LinkedList<>();
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Test Facility", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Testing Facility", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiation", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study Initiated", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completed on", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study completion", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Study amended", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Report Issue Date", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Document Date", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Date", $page.getMainBodyTextBlock()));
stopBoundaries.addAll(RedactionSearchUtility.findTextRangesByStringIgnoreCase("Defitrace", $page.getMainBodyTextBlock()));
entityCreationService.betweenTextRanges(startBoundaries, stopBoundaries, "author", EntityType.ENTITY, document).forEach(entity -> {
entity.apply("DOC.5.1", "Author list found", "n-a");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.2: Study Director on cover page in a paragraph"
when
not TextEntity(type=="author", applied())
$sectionTitle: String() from List.of(
"Study Director"
)
$node: Paragraph(
onPage(1)
&& containsStringIgnoreCase($sectionTitle)
)
then
entityCreationService
.byRegexWithLineBreaksIgnoreCase(
"\\n([\\w\\(\\) .]{5,30})\\n",
"author",
EntityType.ENTITY,
1,
$node
)
.filter(
entity -> !entity.getValue().toLowerCase().contains($sectionTitle.toLowerCase())
)
.forEach(entity -> {
entity.apply("DOC.5.2","Study Director on cover page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.3: Study Director on cover page in a section"
when
not TextEntity(type=="author", applied())
$page: Page(getNumber() == 1)
$sectionTitle: String() from List.of(
"Study Director:",
"Study Director",
"Study Director/Author"
)
$node: Section(
onPage(1)
&& containsStringIgnoreCase($sectionTitle)
)
then
entityCreationService
.byRegexWithLineBreaksIgnoreCase(
$sectionTitle+"\\n([\\w\\(\\) .]{5,30})\\n",
"author",
EntityType.ENTITY,
1,
$node
)
.filter(
entity -> entity.getPages().contains($page)
)
.forEach(entity -> {
entity.apply("DOC.5.3","Study Director on cover page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
entityCreationService
.lineAfterStringIgnoreCase($sectionTitle,"author",EntityType.ENTITY,$node)
.forEach(entity -> {
entity.apply("DOC.5.2","Study Director on cover page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.4: Author in document section with relevant headline"
when
not TextEntity(type == "author", applied())
$hlKeyword: String() from List.of(
"sponsor",
"personnel",
"staff involved",
"study details",
"management of study"
)
$keyword: String() from List.of(
"Study Director",
"Studv Director",
"Study Director:",
"Study Director :",
"stray Birector:",
"Author:",
"Author ",
"Author(s)"
)
$section: Section(
containsStringIgnoreCase($keyword)
&& getHeadline().containsStringIgnoreCase($hlKeyword)
)
then
entityCreationService
.lineAfterStringIgnoreCase(
$keyword,
"author",
EntityType.ENTITY,
$section
)
.findFirst()
.ifPresent(
entity -> {
entity.apply("DOC.5.4","Author found.");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
}
);
end
rule "DOC.5.5: Study Director from signature on compliance statement"
when
not TextEntity(type=="author", applied())
$sectionTitle: String() from List.of(
"Compliance Statement",
"Study Compliance",
"Statement of Compliance",
"Statement of Study Compliance",
"Contributing Scientist",
"Compliance with Good Laboratory Practice",
"Signatures and Approval"
)
$role: String() from List.of("Study Director")
$node: Section(
containsStringIgnoreCase($sectionTitle)
&& containsStringIgnoreCase($role)
&& getFirstPage().getNumber() > 4
)
then
entityCreationService
.byRegexWithLineBreaksIgnoreCase(
"([\\w\\(\\) .,]{5,35})[\\w\\/ ]{0,30}\\n"+$role,
"author",
EntityType.ENTITY,
1,
$node
)
.forEach(entity -> {
entity.apply("DOC.5.5","Study Director on compliance page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.6: Study Director from section with relevant headlining responsibility"
when
not TextEntity(type=="author", applied())
$sectionTitle: String() from List.of("Responsible personnel")
$role: String() from List.of("Study Director")
$node: Section(
getHeadline().containsStringIgnoreCase($sectionTitle)
&& containsStringIgnoreCase($role)
)
$paragraph: Paragraph(
containsStringIgnoreCase($role)
)
then
entityCreationService
.lineAfterStringIgnoreCase(
$role,
"author",
EntityType.ENTITY,
$paragraph
)
.forEach(entity -> {
entity.apply("DOC.5.6","Study Director on compliance page found");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.7: Study Director from section with study director in headline"
when
not TextEntity(type=="author", applied())
$sectionTitle: String() from List.of("Study Director")
$section: Section(
getHeadline().containsStringIgnoreCase($sectionTitle)
)
then
entityCreationService
.bySemanticNodeParagraphsOnly(
$section,
"author",
EntityType.ENTITY
)
.forEach(entity -> {
entity.apply("DOC.5.7","Study Director found by headline");
insert(FileAttribute.builder().label("Author").value(entity.getValue()).build());
});
end
rule "DOC.5.9: Remove skipped authors"
salience 9999
when
$author: TextEntity(type=="author", !applied())
then
$author.removeFromGraph();
end
rule "DOC.14.0: GLP Study"
when
$hlKeywords: String() from List.of(
"Good Laboratory Practice",
"GLP"
)
$headline: Headline(
containsString($hlKeywords)
)
$compliance: String() from List.of(
"conducted in compliance with",
"conducted in accordance with",
"conducted and reported in compliance with",
"carried out in accordance with",
"meets the requirements",
"performed in compliance with"
)
$section: Section(
containsStringIgnoreCase($compliance)
&& getHeadline() == $headline
)
then
entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> {
entity.apply("DOC.14.0", "GLP Study found", "n-a");
insert(FileAttribute.builder().label("GLP").value("TRUE").build());
});
end
rule "DOC.14.1: GLP Study"
when
$compliance: String() from List.of(
"conducted in compliance with",
"conducted in accordance with",
"conducted and reported in compliance with",
"carried out in accordance with",
"meets the requirements",
"performed in compliance with",
"conduct"
)
$principle: String() from List.of(
"40 CFR Part 160",
"40CFR160",
"Good Laboratory Practice",
"FIFRA Good Laboratory Practice Standards"
)
$paragraph: Paragraph(
containsStringIgnoreCase($compliance)
&& containsStringIgnoreCase($principle)
)
then
entityCreationService.betweenStringsIncludeStartAndEndIgnoreCase(
$compliance,
$principle,
"glp_study",
EntityType.ENTITY,
$paragraph
)
.forEach(entity -> {
entity.apply("DOC.14.1", "GLP Study found", "n-a");
insert(FileAttribute.builder().label("GLP").value("TRUE").build());
}
);
end
//------------------------------------ Manual redaction rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
end
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToForce: Image(id == $id)
then
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
retract($force);
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
end
rule "MAN.3.2: Apply image recategorization"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
end
rule "MAN.4.1: Apply legal basis change"
salience 128
when
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
// Rule unit: X.2
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.3
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
// Rule unit: X.4
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
// Rule unit: X.5
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
// Rule unit: X.7
rule "X.7.0: remove all images"
salience 512
when
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
then
$image.remove("X.7.0", "remove all images");
retract($image);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($duplicate);
end
// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end

21
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -133,7 +133,7 @@ rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
// Rule unit: CBI.2
rule "CBI.2.0: Do not redact genitive CBI Author"
when
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), applied())
$entity: TextEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
then
entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
@ -646,6 +646,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
});
end
// Rule unit: CBI.21
rule "CBI.21.0: Redact short Authors section (non vertebrate study)"
when
@ -797,6 +798,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.4
rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
when
@ -1034,6 +1036,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList()));
end
// Rule unit: PII.13
rule "PII.13.0: Add recommendation for PII after Contact Person"
when
@ -1094,12 +1097,12 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
// Rule unit: ETC.3
rule "ETC.3.0: Redact logos (non vertebrate study)"
rule "ETC.3.0: Skip logos (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.redact("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.skip("ETC.3.0", "Logo Found");
end
rule "ETC.3.1: Redact logos (vertebrate study)"
@ -1187,6 +1190,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
$logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: ETC.9
rule "ETC.9.0: Redact skipped impurities"
when
@ -1204,6 +1208,7 @@ rule "ETC.9.1: Redact impurities"
$skippedImpurities.redact("ETC.9.1", "Impurity found", "Article 63(2)(b) of Regulation (EC) No 1107/2009");
end
// Rule unit: ETC.10
rule "ETC.10.0: Redact Product Composition Information"
when
@ -1212,6 +1217,7 @@ rule "ETC.10.0: Redact Product Composition Information"
$compositionInformation.redact("ETC.10.0", "Product Composition Information found", "Article 63(2)(d) of Regulation (EC) No 1107/2009");
end
// Rule unit: ETC.11
rule "ETC.11.0: Recommend first line in table cell with name and address of owner"
when
@ -1223,6 +1229,7 @@ rule "ETC.11.0: Recommend first line in table cell with name and address of owne
.ifPresent(redactionEntity -> redactionEntity.redact("ETC.11.0", "Trial Site owner and address found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1268,7 +1275,8 @@ rule "AI.3.0: Recommend authors from AI as PII"
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, "PII", EntityType.RECOMMENDATION, document));
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1502,12 +1510,11 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -1540,6 +1547,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -66,7 +66,7 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ H rules ------------------------------------
//------------------------------------ Headlines rules ------------------------------------
// Rule unit: H.0
rule "H.0.0: retract table of contents page"
@ -1152,7 +1152,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1394,6 +1394,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

12
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora_components.drl
View File

@ -313,22 +313,13 @@ rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
rule "StudyDesign.0.0: Study design as one block"
when
$oecdNumber: String() from List.of("404", "405", "406", "428", "438", "439", "474", "487")
$oecdNumber: String() from List.of("404", "405", "406", "428", "429", "438", "439", "474", "487")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$studyDesigns: List() from collect (Entity(type == "study_design"))
then
componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
end
rule "StudyDesign.0.1: Study design (Main Study) as one block"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$studyDesigns: List() from collect (Entity(type == "study_design"))
then
componentCreationService.joining("StudyDesign.0.0", "Study_Design_Main_Study", $studyDesigns, " ");
end
rule "Results.0.0: Results and conclusions as joined values"
when
$oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
@ -510,6 +501,7 @@ rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
rule "DefaultComponents.999.0: Create components for all unmapped entities."
salience -999
when
not FileAttribute(label == "OECD Number")
$allEntities: List(!isEmpty()) from collect (Entity())
then
componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -66,7 +66,7 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -163,7 +163,6 @@ rule "MAN.3.0: Apply entity recategorization"
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
salience 128
when
@ -175,7 +174,6 @@ rule "MAN.3.1: Apply entity recategorization of same type"
retract($recategorization);
end
rule "MAN.3.2: Apply image recategorization"
salience 128
when
@ -208,6 +206,8 @@ rule "MAN.4.1: Apply legal basis change"
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0
@ -220,6 +220,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

172
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -84,7 +84,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.3
rule "CBI.3.0: Redacted because Section contains Vertebrate"
rule "CBI.3.0: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
@ -99,7 +99,7 @@ rule "CBI.3.0: Redacted because Section contains Vertebrate"
});
end
rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
rule "CBI.3.1: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
@ -115,7 +115,7 @@ rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
});
end
rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
@ -123,7 +123,7 @@ rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
.forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
end
rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
then
@ -134,7 +134,7 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
// Rule unit: CBI.4
rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section"
rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("vertebrate"),
@ -151,7 +151,7 @@ rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is foun
});
end
rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row"
rule "CBI.4.1: Do not redact Names and Addresses if no_redaction_indicator is found in table row"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("vertebrate"),
@ -172,7 +172,7 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
// Rule unit: CBI.5
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section"
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("redaction_indicator"),
@ -192,7 +192,7 @@ rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also red
});
end
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row"
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
@ -229,7 +229,7 @@ rule "CBI.8.0: Redacted because Section contains must_redact entity"
});
end
rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
rule "CBI.8.1: Redacted because table row contains must_redact entity"
when
$table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
@ -247,7 +247,7 @@ rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
// Rule unit: CBI.9
rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non vertebrate study)"
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -257,10 +257,10 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -270,7 +270,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -286,7 +286,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
// Rule unit: CBI.12
rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
rule "CBI.12.0: Add all cells with Header Author(s) as CBI_author"
salience 1
when
$table: Table(hasHeader("Author(s)") || hasHeader("Author"))
@ -301,7 +301,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
end
rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
rule "CBI.12.1: Do not redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
when
$table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
then
@ -316,7 +316,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
.forEach(authorEntity -> authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -325,7 +325,7 @@ rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at
when
$sponsorEntity: TextEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.apply("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
$sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -350,7 +350,7 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
.toList();
$section.getEntitiesOfType(List.of($keyword, $residueKeyword))
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
@ -372,12 +372,12 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
// Rule unit: CBI.16
rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -385,12 +385,12 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
@ -398,7 +398,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
@ -477,7 +477,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
laboratoryEntity.redact("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(laboratoryEntity);
});
end
@ -491,7 +491,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
then
$pii.apply("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "PII.0.1: Redact all PII (vertebrate study)"
@ -499,7 +499,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
then
$pii.apply("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -510,7 +510,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@ -519,7 +519,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -549,12 +549,12 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
.forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)"
rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
@ -577,12 +577,12 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
// Rule unit: PII.6
rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@ -591,10 +591,10 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.1: redact line between contact keywords"
rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
@ -603,7 +603,7 @@ rule "PII.6.1: redact line between contact keywords"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -623,10 +623,10 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(getHeadline().containsString("applicant") ||
@ -641,12 +641,12 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.8
rule "PII.8.0: Redact contact information if producer is found"
rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@ -661,10 +661,10 @@ rule "PII.8.0: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
.forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.8.1: Redact contact information if producer is found"
rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
@ -679,45 +679,27 @@ rule "PII.8.1: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
.forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
.forEach(authorEntity -> authorEntity.redact("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -727,7 +709,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -749,7 +731,7 @@ rule "ETC.1.0: Redact Purity"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.apply("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
.forEach(entity -> entity.redact("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
end
@ -759,7 +741,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$signature.redact("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.2.1: Redact signatures (vertebrate study)"
@ -767,25 +749,25 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.apply("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: ETC.3
rule "ETC.3.0: Redact logos (vertebrate study)"
rule "ETC.3.0: Skip logos (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.apply("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.skip("ETC.3.0", "Logo Found");
end
rule "ETC.3.1: Redact logos (non vertebrate study)"
rule "ETC.3.1: Redact logos (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.apply("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$logo.redact("ETC.3.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -794,7 +776,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
when
$dossierRedaction: TextEntity(type == "dossier_redaction")
then
$dossierRedaction.apply("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -818,14 +800,14 @@ rule "ETC.6.0: Redact CAS Number"
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
.forEach(redactionEntity -> redactionEntity.redact("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
// Rule unit: ETC.7
rule "ETC.7.0: Guidelines FileAttributes"
when
$section: Section(!hasTables(), (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
$section: Section(!hasTables(), containsAnyString("DATA REQUIREMENT(S):", "TEST GUIDELINE(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
then
RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
@ -840,7 +822,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)"
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.FORMULA)
then
$logo.apply("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.8.1: Redact formulas (non vertebrate study)"
@ -848,14 +830,14 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.FORMULA)
then
$logo.apply("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
rule "AI.0.0: add all NER Entities of type CBI_author"
rule "AI.0.0: Add all NER Entities of type CBI_author"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@ -866,7 +848,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
// Rule unit: AI.1
rule "AI.1.0: combine and add NER Entities as CBI_address"
rule "AI.1.0: Combine and add NER Entities as CBI_address"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
@ -875,7 +857,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1020,7 +1002,7 @@ rule "MAN.4.1: Apply legal basis change"
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
@ -1032,7 +1014,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
// Rule unit: X.1
rule "X.1.0: merge intersecting Entities of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@ -1048,7 +1030,7 @@ rule "X.1.0: merge intersecting Entities of same type"
// Rule unit: X.2
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@ -1061,7 +1043,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
// Rule unit: X.3
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@ -1073,7 +1055,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
// Rule unit: X.4
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -1086,7 +1068,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
// Rule unit: X.5
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -1098,7 +1080,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when contained by entity of type ENTITY"
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -1113,7 +1095,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -1124,7 +1106,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: remove duplicate FileAttributes"
rule "FA.1.0: Remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
@ -1146,6 +1128,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

90
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -60,16 +60,40 @@ global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: TextEntity(type == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
//------------------------------------ PII rules ------------------------------------
// Rule unit: PII.0
rule "PII.0.0: Redact all PII (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$pii: TextEntity(type == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
rule "AI.0.0: add all NER Entities of type CBI_author"
rule "AI.0.0: Add all NER Entities of type CBI_author"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
@ -78,28 +102,8 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "CBI.0.0: Always redact CBI_author"
when
$cbiAuthor: TextEntity(type == "CBI_author", entityType == EntityType.ENTITY)
then
$cbiAuthor.apply("CBI.0.0", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "PII.0.0: Always redact PII"
when
$cbiAuthor: TextEntity(type == "PII", entityType == EntityType.ENTITY)
then
$cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -157,7 +161,6 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@ -166,10 +169,10 @@ rule "MAN.2.0: Apply force redaction"
$entityToForce.getManualOverwrite().addChange($force);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
@ -178,6 +181,7 @@ rule "MAN.2.1: Apply force redaction to images"
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
retract($force);
end
@ -185,9 +189,9 @@ rule "MAN.2.1: Apply force redaction to images"
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
@ -196,16 +200,14 @@ rule "MAN.3.0: Apply entity recategorization"
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply image recategorization"
rule "MAN.3.1: Apply entity recategorization of same type"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
end
@ -233,7 +235,7 @@ rule "MAN.4.1: Apply legal basis change"
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
@ -245,7 +247,7 @@ rule "X.0.0: remove Entity contained by Entity of same type"
// Rule unit: X.1
rule "X.1.0: merge intersecting Entities of same type"
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
@ -261,7 +263,7 @@ rule "X.1.0: merge intersecting Entities of same type"
// Rule unit: X.2
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
@ -274,7 +276,7 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
// Rule unit: X.3
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
@ -286,7 +288,7 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
// Rule unit: X.4
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -299,7 +301,7 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
// Rule unit: X.5
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -311,7 +313,7 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
// Rule unit: X.6
rule "X.6.0: remove Entity of lower rank, when contained by by entity of type ENTITY"
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
@ -326,7 +328,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -337,7 +339,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: remove duplicate FileAttributes"
rule "FA.1.0: Remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
@ -359,6 +361,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

56
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -66,23 +66,9 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end
// --------------------------------------- Your rules below this line --------------------------------------------------
//------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0
rule "TAB.0.0: Study Type File Attribute"
when
not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
@ -115,6 +101,8 @@ rule "TAB.0.1: Guidelines"
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
end
// Rule unit: TAB.1
rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -126,6 +114,8 @@ rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
.ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
end
// Rule unit: TAB.2
rule "TAB.2.0: Individual row extraction (Clinical Signs)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -138,6 +128,8 @@ rule "TAB.2.0: Individual row extraction (Clinical Signs)"
.ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
end
// Rule unit: TAB.3
rule "TAB.3.0: Individual column extraction (Strain)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -151,6 +143,8 @@ rule "TAB.3.0: Individual column extraction (Strain)"
.forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
end
// Rule unit: TAB.4
rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -177,6 +171,8 @@ rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
.forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
end
// Rule unit: TAB.5
rule "TAB.5.0: Targeted cell extraction"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -190,6 +186,8 @@ rule "TAB.5.0: Targeted cell extraction"
.ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
end
// Rule unit: TAB.6
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
when
$section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
@ -202,6 +200,8 @@ rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
.ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
end
// Rule unit: TAB.7
rule "TAB.7.0: Indicator (Species)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
@ -215,7 +215,8 @@ rule "TAB.7.0: Indicator (Species)"
.ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -433,12 +434,11 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -447,7 +447,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
// Rule unit: X.7
rule "X.7.0: remove all images"
rule "X.7.0: Remove all images"
salience 512
when
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
@ -469,3 +469,19 @@ rule "FA.1.0: Remove duplicate FileAttributes"
retract($duplicate);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl
View File

@ -101,3 +101,26 @@ rule "EntityBasedExtr.2.4: Cells containing dose for survived males"
then
componentCreationService.joiningFromSameTableRow("EntityBasedExtr.2.4", "2.4 Entity-Based Values", $tableValues);
end
rule "DefaultComponents.999.0: Create components for all unmapped entities."
salience -999
when
not FileAttribute(label == "OECD Number")
$allEntities: List(!isEmpty()) from collect (Entity())
then
componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
end
//------------------------------------ Component merging rules ------------------------------------
/*
rule "X.0.0: merge duplicate component references"
when
$first: Component()
$duplicate: Component(this != $first, name == $first.name, value == $first.value)
then
$first.getReferences().addAll($duplicate.getReferences());
retract($duplicate);
end
*/

56
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -66,23 +66,9 @@ query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end
// --------------------------------------- Your rules below this line --------------------------------------------------
//------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0
rule "TAB.0.0: Study Type File Attribute"
when
not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
@ -116,20 +102,21 @@ rule "TAB.0.1: Guidelines"
end
// Rule unit: TAB.6
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
when
$section: Section(containsString("Maximum occurrence"))
$table: Table() from $section.streamChildren().toList()
TableCell(containsWordIgnoreCase("water"), $row: row) from $table.streamTableCells().toList()
$test: TableCell($row == row) from $table.streamTableCells().toList()
$section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
$table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
$femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
then
System.out.println("AAAA: " + $test);
entityCreationService.bySemanticNode($test, "test", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.6.0", "Some test stuff"));
entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -363,12 +350,11 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -377,7 +363,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
// Rule unit: X.7
rule "X.7.0: remove all images"
rule "X.7.0: Remove all images"
salience 512
when
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
@ -399,3 +385,19 @@ rule "FA.1.0: Remove duplicate FileAttributes"
retract($duplicate);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

8
redaction-service-v1/rules-management/build.gradle.kts
View File

@ -16,6 +16,14 @@ repositories {
}
}
sourceSets {
test {
resources {
srcDirs("src/main/resources", "src/test/ressources") // add both so test can access the all_rules_file
}
}
}
dependencies {
implementation(project(":redaction-service-server-v1"))
implementation("org.projectlombok:lombok:1.18.28")

56
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactory.java
View File

@ -4,6 +4,7 @@ import static java.util.Collections.emptySet;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@ -23,6 +24,7 @@ import lombok.experimental.UtilityClass;
public class RuleFileFactory {
public String createFileFromIdentifiers(String identifiers, ApplicationType applicationType) {
if (identifiers.isBlank() || identifiers.isEmpty()) {
return createFileFromIdentifiers(emptySet(), applicationType);
}
@ -35,6 +37,7 @@ public class RuleFileFactory {
return createFileFromIdentifiers(identifiers, true, applicationType);
}
public String createFileFromIdentifiers(Set<RuleIdentifier> identifiers, boolean includeDefaults, ApplicationType applicationType) {
if (includeDefaults) {
@ -42,12 +45,12 @@ public class RuleFileFactory {
}
RuleFileBluePrint bluePrint = RuleFileParser.buildBluePrintFromAllRulesFile(applicationType);
RuleFileBluePrint filteredBluePrint = bluePrint.buildFilteredBluePrintByRuleIdentifiers(identifiers);
return buildRuleFile(filteredBluePrint);
return buildRuleString(filteredBluePrint);
}
@SneakyThrows
public String buildRuleFile(RuleFileBluePrint bluePrint) {
public String buildRuleString(RuleFileBluePrint bluePrint) {
try (var templateInputStream = RuleManagementResources.getTemplateInputStream()) {
String template = new String(templateInputStream.readAllBytes(), StandardCharsets.UTF_8);
@ -66,6 +69,8 @@ public class RuleFileFactory {
private String buildBluePrintWithTemplateRuleOrder(RuleFileBluePrint bluePrint, List<RuleType> ruleOrder) {
Set<RuleType> additionalRuleTypes = bluePrint.ruleClasses().stream().map(RuleClass::ruleType).filter(ruleType -> !ruleOrder.contains(ruleType)).collect(Collectors.toSet());
StringBuilder sb = new StringBuilder();
sb.append(bluePrint.imports());
sb.append("\n\n");
@ -76,29 +81,40 @@ public class RuleFileFactory {
sb.append(bluePrint.queries());
sb.append("\n\n");
for (RuleType ruleBlockType : ruleOrder) {
if (!bluePrint.ruleClassExists(ruleBlockType)) {
if (ruleBlockType.isWildCard()) {
additionalRuleTypes.stream().sorted(Comparator.comparing(RuleType::name)).forEach(ruleType -> writeRuleClass(bluePrint, ruleType, sb));
continue;
}
sb.append("//------------------------------------ ");
sb.append(ruleBlockType);
sb.append(" rules ------------------------------------");
sb.append("\n\n");
RuleClass ruleClass = bluePrint.findRuleClassByType(ruleBlockType);
for (RuleUnit unit : ruleClass.ruleUnits()) {
if (unit.rules().isEmpty()) {
continue;
}
sb.append("// ");
sb.append(unit.rules().get(0).identifier().toRuleUnitString());
sb.append("\n");
unit.rules().forEach(rule -> {
sb.append(rule.code());
sb.append("\n\n");
});
sb.append("\n");
if (bluePrint.findRuleClassByType(ruleBlockType).isEmpty()) {
continue;
}
writeRuleClass(bluePrint, ruleBlockType, sb);
}
return sb.toString().trim() + "\n";
}
private static void writeRuleClass(RuleFileBluePrint bluePrint, RuleType ruleType, StringBuilder sb) {
sb.append("//------------------------------------ ");
sb.append(ruleType);
sb.append(" rules ------------------------------------");
sb.append("\n\n");
RuleClass ruleClass = bluePrint.findRuleClassByType(ruleType).orElseThrow();
List<RuleUnit> sortedRuleUnits = ruleClass.ruleUnits().stream().sorted(Comparator.comparingInt(RuleUnit::unit)).toList();
for (RuleUnit unit : sortedRuleUnits) {
if (unit.rules().isEmpty()) {
continue;
}
sb.append("// ");
sb.append(unit.rules().get(0).identifier().toRuleUnitString());
sb.append("\n");
unit.rules().stream().sorted(Comparator.comparingInt(rule -> rule.identifier().id())).forEach(rule -> {
sb.append(rule.code());
sb.append("\n\n");
});
sb.append("\n");
}
}
}

13
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileParser.java
View File

@ -94,4 +94,17 @@ public class RuleFileParser {
return parseRuleIdentifiersFromFile(file.toString());
}
/**
* Creates a BluePrint from all redact manager, documine and component rule files
*
* @return RuleFileBluePrint containing all rules from any all rule files
*/
public static RuleFileBluePrint buildBluePrintFromAllRuleFiles() {
RuleFileBluePrint bluePrint = buildBluePrintFromAllRulesFile(ApplicationType.DM);
bluePrint.addAllRulesFromBluePrint(buildBluePrintFromAllRulesFile(ApplicationType.RM));
return bluePrint;
}
}

43
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleFileMigrator.java Normal file
View File

@ -0,0 +1,43 @@
package com.knecon.fforesight.utility.rules.management.migration;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
import com.knecon.fforesight.utility.rules.management.models.BasicRule;
import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
import lombok.SneakyThrows;
import lombok.experimental.UtilityClass;
/**
* This class is the primary class for migrating Files with the 4.0.0 format to newer versions. This assumes RuleIdentifiers are already formatted.
* It will search for any rule with the same Identifier in the base and replace it with the new version. If none is found, the rule is written as is.
*/
@UtilityClass
public class RuleFileMigrator {
@SneakyThrows
public void migrateFile(File ruleFile) {
RuleFileBluePrint ruleFileBluePrint = RuleFileParser.buildBluePrintFromRulesString(RuleFileIO.getRulesString(ruleFile.getAbsolutePath()));
RuleFileBluePrint combinedBluePrint = RuleFileParser.buildBluePrintFromAllRuleFiles();
for (BasicRule ruleToReplace : ruleFileBluePrint.getAllRules()) {
List<BasicRule> rulesToAdd = combinedBluePrint.findRuleByIdentifier(ruleToReplace.identifier());
ruleFileBluePrint.removeRule(ruleToReplace.identifier());
rulesToAdd.forEach(ruleFileBluePrint::addRule);
}
String migratedRulesString = RuleFileFactory.buildRuleString(ruleFileBluePrint);
String migratedFilePath = ruleFile.getAbsolutePath();
try (var out = new FileOutputStream(migratedFilePath)) {
out.write(migratedRulesString.getBytes(StandardCharsets.UTF_8));
}
}
}

14
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/migration/RuleIdentifierMigrator.java
View File

@ -46,7 +46,7 @@ public class RuleIdentifierMigrator {
bluePrint = migrateMatchedRuleForAllRules(bluePrint);
String ruleString = RuleFileFactory.buildRuleFile(bluePrint);
String ruleString = RuleFileFactory.buildRuleString(bluePrint);
try (var out = new FileOutputStream("/tmp/all_redact_manager_rules.drl")) {
out.write(ruleString.getBytes(StandardCharsets.UTF_8));
}
@ -78,8 +78,11 @@ public class RuleIdentifierMigrator {
String redactionReason = findByRegex("\\.setRedactionReason\\(\"(.*)\"\\)", basicRule.code(), 1);
String legalBasis = findByRegex("\\.setLegalBasis\\(\"(.*)\"\\)", basicRule.code(), 1);
String migratedCode = basicRule.code().replaceAll("\\.addMatchedRule\\(.*\\)", Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
migratedCode = migratedCode.replaceAll("\\.setMatchedRule\\(.*\\)", Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
String migratedCode = basicRule.code()
.replaceAll("\\.addMatchedRule\\(.*\\)",
Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
migratedCode = migratedCode.replaceAll("\\.setMatchedRule\\(.*\\)",
Matcher.quoteReplacement(String.format(".addMatchedRule(\"%s\", \"%s\", \"%s\")", basicRule.identifier().toString(), redactionReason, legalBasis)));
migratedCode = migratedCode.replaceAll("\\.setRedactionReason\\(\".*\"\\)", "");
migratedCode = migratedCode.replaceAll("\\.setLegalBasis\\(\".*\"\\)", "");
migratedCode = migratedCode.replaceAll("\\$entity;\n", "");
@ -89,6 +92,7 @@ public class RuleIdentifierMigrator {
return new BasicRule(basicRule.identifier(), basicRule.name(), migratedCode);
}
private static String findByRegex(String regex, String searchText, int group) {
Pattern pattern = Pattern.compile(regex);
@ -113,8 +117,8 @@ public class RuleIdentifierMigrator {
public void migrateIdentifier(RuleIdentifier oldIdentifier, RuleIdentifier newIdentifier, RuleFileBluePrint bluePrint, List<OldRulesParser.OldRulesCsvRecord> records) {
BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type())
.findRuleUnitByInteger(oldIdentifier.unit())
BasicRule oldRule = bluePrint.findRuleClassByType(oldIdentifier.type()).orElseThrow()
.findRuleUnitByInteger(oldIdentifier.unit()).orElseThrow()
.rules()
.stream()
.filter(rule -> rule.identifier().equals(oldIdentifier))

15
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleClass.java
View File

@ -1,22 +1,23 @@
package com.knecon.fforesight.utility.rules.management.models;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
public record RuleClass(RuleType ruleType, List<RuleUnit> ruleUnits) {
public RuleUnit findRuleUnitByInteger(Integer unit) {
public Optional<RuleUnit> findRuleUnitByInteger(Integer unit) {
return ruleUnits.stream()
.filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(String.format("RuleUnit %d does not exist in class %s", unit, this)));
return ruleUnits.stream().filter(ruleUnit -> Objects.equals(ruleUnit.unit(), unit)).findFirst();
}
public boolean ruleUnitExists(Integer unit) {
public RuleUnit createNewRuleUnit(Integer unit) {
return ruleUnits.stream().anyMatch(ruleUnit -> Objects.equals(ruleUnit.unit(), unit));
var ruleUnit = new RuleUnit(unit, new LinkedList<>());
ruleUnits.add(ruleUnit);
return ruleUnit;
}
}

121
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleFileBluePrint.java
View File

@ -1,76 +1,79 @@
package com.knecon.fforesight.utility.rules.management.models;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
public record RuleFileBluePrint(String imports, String globals, String queries, List<RuleClass> ruleClasses) {
public RuleClass findRuleClassByType(RuleType ruleType) {
public void removeRule(RuleIdentifier ruleIdentifier) {
findRuleClassByType(ruleIdentifier.type()).ifPresent(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit()).ifPresent(ruleUnit -> {
ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier));
if (ruleUnit.rules().isEmpty()) {
ruleClass.ruleUnits().remove(ruleUnit);
}
if (ruleClass.ruleUnits().isEmpty()) {
ruleClasses().remove(ruleClass);
}
}));
return ruleClasses.stream()
.filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType))
.findFirst()
.orElseThrow(() -> new IllegalArgumentException(String.format("RuleType %s does not exist in this BluePrint %s", ruleType.name(), this)));
}
public boolean ruleClassExists(RuleType ruleType) {
public Optional<RuleClass> findRuleClassByType(RuleType ruleType) {
return ruleClasses.stream().anyMatch(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType));
return ruleClasses.stream().filter(ruleClass -> Objects.equals(ruleClass.ruleType(), ruleType)).findFirst();
}
public Set<BasicRule> getAllRules() {
return ruleClasses().stream().map(RuleClass::ruleUnits).flatMap(Collection::stream).map(RuleUnit::rules).flatMap(Collection::stream).collect(Collectors.toSet());
}
/**
* Adds all rules from a given RuleFileBluePrint to the current instance.
*
* @param ruleFileBluePrint The RuleFileBluePrint that contains the rules to be added.
* @throws IllegalArgumentException If the RuleFileBluePrints have non-matching rules with the same identifier and cannot be merged.
*/
public void addAllRulesFromBluePrint(RuleFileBluePrint ruleFileBluePrint) {
Set<RuleIdentifier> newRuleIdentifiers = ruleFileBluePrint.getAllRuleIdentifiers();
Set<RuleIdentifier> existingRuleIdentifiers = this.getAllRuleIdentifiers();
Set<RuleIdentifier> duplicatedRuleIdentifiers = existingRuleIdentifiers.stream().filter(newRuleIdentifiers::contains).collect(Collectors.toSet());
for (RuleIdentifier duplicatedRuleIdentifier : duplicatedRuleIdentifiers) {
List<BasicRule> thisRules = findRuleByIdentifier(duplicatedRuleIdentifier);
List<BasicRule> otherRules = ruleFileBluePrint.findRuleByIdentifier(duplicatedRuleIdentifier);
assert thisRules.size() == 1;
assert otherRules.size() == 1;
if (!thisRules.get(0).equals(otherRules.get(0))) {
throw new IllegalArgumentException("RuleFileBluePrints have non matching rules with the same identifier, cannot be merged!");
}
}
newRuleIdentifiers.removeAll(existingRuleIdentifiers);
newRuleIdentifiers.forEach(identifierToAdd -> ruleFileBluePrint.findRuleByIdentifier(identifierToAdd).forEach(this::addRule));
}
public List<BasicRule> findRuleByIdentifier(RuleIdentifier ruleIdentifier) {
if (Objects.isNull(ruleIdentifier.unit())) {
return findRuleClassByType(ruleIdentifier.type()).ruleUnits().stream().map(RuleUnit::rules).flatMap(Collection::stream).toList();
}
return findRuleClassByType(ruleIdentifier.type()).findRuleUnitByInteger(ruleIdentifier.unit())
.rules()
.stream()
.filter(rule -> rule.identifier().matches(ruleIdentifier))
.toList();
}
public void addRule(BasicRule rule) {
RuleClass ruleClass;
if (ruleClassExists(rule.identifier().type())) {
ruleClass = findRuleClassByType(rule.identifier().type());
} else {
ruleClass = new RuleClass(rule.identifier().type(), new LinkedList<>());
ruleClasses.add(ruleClass);
}
RuleUnit ruleUnit;
if (ruleClass.ruleUnitExists(rule.identifier().unit())) {
ruleUnit = ruleClass.findRuleUnitByInteger(rule.identifier().unit());
} else {
ruleUnit = new RuleUnit(rule.identifier().unit(), new LinkedList<>());
ruleClass.ruleUnits().add(ruleUnit);
}
ruleUnit.rules().add(rule);
}
public void removeRule(RuleIdentifier ruleIdentifier) {
RuleClass ruleClass = findRuleClassByType(ruleIdentifier.type());
RuleUnit ruleUnit = ruleClass.findRuleUnitByInteger(ruleIdentifier.unit());
ruleUnit.rules().removeIf(rule -> rule.identifier().matches(ruleIdentifier));
if (ruleUnit.rules().isEmpty()) {
ruleClass.ruleUnits().remove(ruleUnit);
}
if (ruleClass.ruleUnits().isEmpty()) {
ruleClasses().remove(ruleClass);
return findRuleClassByType(ruleIdentifier.type()).map(ruleClass -> ruleClass.ruleUnits().stream().flatMap(ruleUnit -> ruleUnit.rules().stream()).toList())
.orElse(Collections.emptyList());
}
return findRuleClassByType(ruleIdentifier.type())//
.map(ruleClass -> ruleClass.findRuleUnitByInteger(ruleIdentifier.unit())//
.map(ruleUnit -> ruleUnit.rules().stream().filter(rule -> rule.identifier().matches(ruleIdentifier)).toList())//
.orElse(Collections.emptyList()))//
.orElse(Collections.emptyList());
}
@ -86,6 +89,16 @@ public record RuleFileBluePrint(String imports, String globals, String queries,
}
public void addRule(BasicRule rule) {
RuleClass ruleClass = findRuleClassByType(rule.identifier().type()).orElseGet(() -> createNewRuleClass(rule));
RuleUnit ruleUnit = ruleClass.findRuleUnitByInteger(rule.identifier().unit()).orElseGet(() -> ruleClass.createNewRuleUnit(rule.identifier().unit()));
ruleUnit.rules().add(rule);
}
public RuleFileBluePrint buildFilteredBluePrintByRuleIdentifiers(Set<RuleIdentifier> identifiers) {
RuleFileBluePrint filteredBluePrint = new RuleFileBluePrint(imports(), globals(), queries(), new LinkedList<>());
@ -97,4 +110,12 @@ public record RuleFileBluePrint(String imports, String globals, String queries,
return filteredBluePrint;
}
private RuleClass createNewRuleClass(BasicRule rule) {
var newClass = new RuleClass(rule.identifier().type(), new LinkedList<>());
ruleClasses.add(newClass);
return newClass;
}
}

115
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/models/RuleType.java
View File

@ -1,83 +1,54 @@
package com.knecon.fforesight.utility.rules.management.models;
public enum RuleType {
SYN {
@Override
public String toString() {
import java.util.Map;
return "Syngenta specific";
}
},
CBI,
PII,
ETC {
@Override
public String toString() {
import lombok.AccessLevel;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.experimental.FieldDefaults;
return "Other";
}
},
AI,
MAN {
@Override
public String toString() {
@AllArgsConstructor
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
public class RuleType {
return "Manual redaction";
}
},
X {
@Override
public String toString() {
static Map<String, String> fullNameMap = Map.of("SYN", "Syngenta specific",//
"ETC", "Other",//
"MAN", "Manual changes",//
"X", "Entity merging",//
"FA", "File attributes",//
"LDS", "Local dictionary search",//
"TAB", "Table extraction",//
"H", "Headlines",//
"DOC", "General documine");
return "Entity merging";
}
},
FA {
@Override
public String toString() {
return "File attributes";
}
},
LDS {
@Override
public String toString() {
return "Local dictionary search";
}
},
H {
@Override
public String toString() {
return "H";
}
},
DOC {
@Override
public String toString() {
return "General documine";
}
};
@EqualsAndHashCode.Include
String name;
public static RuleType fromString(String value) {
value = value.replaceAll("\r","");
return switch (value) {
case "SYN" -> SYN;
case "CBI" -> CBI;
case "PII" -> PII;
case "ETC" -> ETC;
case "AI" -> AI;
case "MAN" -> MAN;
case "X" -> X;
case "FA" -> FA;
case "LDS" -> LDS;
case "H" -> H;
case "DOC" -> DOC;
default -> throw new IllegalStateException("Unexpected value: " + value);
};
value = value.replaceAll("\r", "");
return new RuleType(value);
}
public String name() {
return name;
}
public boolean isWildCard() {
return name.equals("*");
}
@Override
public String toString() {
return fullNameMap.getOrDefault(name, name);
}
}

6
redaction-service-v1/rules-management/src/main/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParser.java
View File

@ -31,7 +31,6 @@ import com.knecon.fforesight.utility.rules.management.models.BasicRule;
import com.knecon.fforesight.utility.rules.management.models.OldRule;
import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
import com.knecon.fforesight.utility.rules.management.models.RuleType;
import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
import lombok.SneakyThrows;
@ -106,10 +105,7 @@ public class OldRulesParser {
String.format("%s: %s", value, value.stream().map(bluePrint::findRuleByIdentifier).flatMap(Collection::stream).map(BasicRule::name).toList())));
return Stream.concat(//
Stream.of(RuleIdentifier.fromRuleType(RuleType.X),
RuleIdentifier.fromRuleType(RuleType.FA),
RuleIdentifier.fromRuleType(RuleType.LDS),
RuleIdentifier.fromRuleType(RuleType.MAN)),//
Stream.of(RuleIdentifier.fromString("X"), RuleIdentifier.fromString("FA"), RuleIdentifier.fromString("LDS"), RuleIdentifier.fromString("MAN")),//
translationPairs.values().stream().flatMap(Collection::stream))
.map(ruleIdentifier -> new RuleIdentifier(ruleIdentifier.type(), ruleIdentifier.unit(), null))
.collect(Collectors.toSet());

10
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1094,15 +1094,15 @@ rule "ETC.2.1: Redact signatures (vertebrate study)"
// Rule unit: ETC.3
rule "ETC.3.0: Redact logos (vertebrate study)"
rule "ETC.3.0: Skip logos (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.redact("ETC.3.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.skip("ETC.3.0", "Logo Found");
end
rule "ETC.3.1: Redact logos (non vertebrate study)"
rule "ETC.3.1: Redact logos (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$logo: Image(imageType == ImageType.LOGO)
@ -1507,7 +1507,7 @@ rule "X.6.1: remove Entity of higher rank, when intersected by entity of type EN
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
@ -1540,6 +1540,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

151
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1149,6 +1149,155 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
.forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
end
//------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0
rule "TAB.0.0: Study Type File Attribute"
when
not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
&& containsAnyString("OECD", "EPA", "OPPTS"))
then
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
end
rule "TAB.0.1: Guidelines"
when
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
then
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
end
// Rule unit: TAB.1
rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Full Table"))
$table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
$tableCell: TableCell(!header) from $table.streamTableCells().toList()
then
entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
end
// Rule unit: TAB.2
rule "TAB.2.0: Individual row extraction (Clinical Signs)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Individual Rows Extraction"))
$table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
$tableCell: TableCell($row == row) from $table.streamTableCells().toList()
then
entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
end
// Rule unit: TAB.3
rule "TAB.3.0: Individual column extraction (Strain)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Individual Column"))
$table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
then
$table.streamTableCellsWithHeader("Sex")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
end
// Rule unit: TAB.4
rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Combined Columns"))
$table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
then
$table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
end
rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Combined Columns"))
$table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
then
$table.streamTableCellsWithHeader("Mortality")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
end
// Rule unit: TAB.5
rule "TAB.5.0: Targeted cell extraction"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Value Extraction"))
$table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
$dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
then
entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
end
// Rule unit: TAB.6
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
when
$section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
$table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
$femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
then
entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
end
// Rule unit: TAB.7
rule "TAB.7.0: Indicator (Species)"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Entity-Based"))
$table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
$cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
then
entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
.ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
end
//------------------------------------ Manual redaction rules ------------------------------------
@ -1392,6 +1541,6 @@ rule "LDS.0.0: Run local dictionary search"
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
});
end

2
redaction-service-v1/rules-management/src/main/resources/order_template.txt
View File

@ -1,8 +1,10 @@
H
DOC
TAB
SYN
CBI
PII
*
ETC
AI
MAN

20
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileBluePrintMergingTest.java Normal file
View File

@ -0,0 +1,20 @@
package com.knecon.fforesight.utility.rules.management;
import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
import com.knecon.fforesight.utility.rules.management.models.RuleFileBluePrint;
import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
public class RuleFileBluePrintMergingTest {
@Test
public void testBothRuleFilesCanBeMerged() {
RuleFileBluePrint combined = RuleFileParser.buildBluePrintFromAllRuleFiles();
assertEquals(1, combined.findRuleByIdentifier(RuleIdentifier.fromString("X.0.0")).size());
}
}

56
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/RuleFileMigrationTest.java Normal file
View File

@ -0,0 +1,56 @@
package com.knecon.fforesight.utility.rules.management;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.utility.rules.management.migration.RuleFileMigrator;
import lombok.SneakyThrows;
/**
* This test may be used to migrate dossier-templates and redaction-service ruleFiles all at once
* It loads the existing ruleFiles looks if there is a different version present in the rules-management. If it is present it is replaced by the rules-management version.
* From these blueprints it overwrites the existing rule files. Which means the order of rules may change according to order_template.txt.
* In comparison to the previous migration code this keeps Rules that are not present in the rules-management untouched. The order of these unknown rules may be adapted using the '*' symbol in the order-template.txt.
* The test does not add any additional rules using the default identifier lists.
* The test does not care about the applicationType, since it looks in both RM and DM all-rules files.
* It does not migrate Component Rules yet.
*/
public class RuleFileMigrationTest {
// Put your redaction service drools paths and dossier-templates paths both RM and DM here
static final List<String> ruleFileDirs = List.of(
"/home/kschuettler/iqser/redaction/redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools",
"/home/kschuettler/iqser/fforesight/dossier-templates-v2/",
"/home/kschuettler/iqser/redaction/dossier-templates-v2/");
@Test
@SneakyThrows
@Disabled
void migrateAllEntityRules() {
for (String ruleFileDir : ruleFileDirs) {
Files.walk(Path.of(ruleFileDir)).filter(this::isEntityRuleFile).map(Path::toFile).peek(System.out::println).forEach(RuleFileMigrator::migrateFile);
}
}
private boolean isEntityRuleFile(Path path) {
File ruleFile = path.toFile();
if (!ruleFile.isFile()) {
return false;
}
if (!ruleFile.toString().endsWith(".drl")) {
return false;
}
return !ruleFile.getName().equals("componentRules.drl") && !ruleFile.getName().endsWith("_components.drl") && !ruleFile.getName().endsWith("OLD.drl");
}
}

106
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java
View File

@ -1,6 +1,5 @@
package com.knecon.fforesight.utility.rules.management.factory;
import static java.util.stream.Collectors.toList;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.BufferedWriter;
@ -13,20 +12,14 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.knecon.fforesight.utility.rules.management.RuleManagementResources;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileFactory;
import com.knecon.fforesight.utility.rules.management.factory.RuleFileParser;
import com.knecon.fforesight.utility.rules.management.models.ApplicationType;
import com.knecon.fforesight.utility.rules.management.models.RuleIdentifier;
import com.knecon.fforesight.utility.rules.management.utils.RuleFileIO;
@ -44,22 +37,23 @@ class RuleFileFactoryTest {
migrate(ApplicationType.RM, dossierTemplatesRepo + "dev", dossierTemplatesRepo + "docu", dossierTemplatesRepo + "qa");
}
@Test
@SneakyThrows
@Disabled
public void migrateDocumineDossierTemplatesRepoToNewestRules() {
String dossierTemplatesRepo = "/Users/aisvoran/dev/documine/dossier-templates-v2/dev/";
migrate(ApplicationType.DM, dossierTemplatesRepo + "Flora", dossierTemplatesRepo + "Basf-Demo");
}
@SneakyThrows
private void migrate(ApplicationType applicationType, String... paths) {
Arrays.stream(paths).forEach(path -> {
try {
Stream.of(Files.walk(Path.of(path)))
.flatMap(Function.identity())
.filter(p -> p.getFileName().toString().equals("rules.drl"))//
Stream.of(Files.walk(Path.of(path))).flatMap(Function.identity()).filter(p -> p.getFileName().toString().equals("rules.drl"))//
.map(Path::toFile)//
.forEach(e -> migrateFile(e, applicationType));
} catch (IOException e) {
@ -76,9 +70,8 @@ class RuleFileFactoryTest {
Set<RuleIdentifier> identifiers = RuleFileParser.parseRuleIdentifiersFromFile(oldRulesFile);
String newRulesString = RuleFileFactory.createFileFromIdentifiers(identifiers, applicationType);
try (FileOutputStream out = new FileOutputStream(oldRulesFile);
OutputStreamWriter outWrite = new OutputStreamWriter(out, StandardCharsets.UTF_8);
BufferedWriter writer = new BufferedWriter(outWrite)) {
try (FileOutputStream out = new FileOutputStream(oldRulesFile); OutputStreamWriter outWrite = new OutputStreamWriter(out,
StandardCharsets.UTF_8); BufferedWriter writer = new BufferedWriter(outWrite)) {
writer.write(newRulesString);
}
}
@ -93,91 +86,6 @@ class RuleFileFactoryTest {
}
@Test
@Disabled
void generateRules() {
String stem = """
rule "DefaultComponents.4.klasjfidklasjf: Test Guideline 1"
when
$guidelineNumber: Entity(type == "oecd_guideline_number", klasjfnumberklasjf)
$guidelineYear: Entity(type == "oecd_guideline_year", klasjfyearklasjf)
then
componentCreationService.createComponent(
"DefaultComponents.4.klasjfidklasjf",
"Test_Guidelines_1",
"klasjfguidelineklasjf",
"OECD Number and guideline year mapped!"
);
end
""";
Map<List<String>, String> guidelineMapping = new HashMap<>();
guidelineMapping.put(List.of("425", "2008"), "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)");
guidelineMapping.put(List.of("425", "2001"), "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)");
guidelineMapping.put(List.of("402", "2017"), "Nº 402: Acute Dermal Toxicity (09/10/2017)");
guidelineMapping.put(List.of("402", "1987"), "Nº 402: Acute Dermal Toxicity (24/02/1987)");
guidelineMapping.put(List.of("403", "2009"), "Nº 403: Acute Inhalation Toxicity (08/09/2009)");
guidelineMapping.put(List.of("403", "1981"), "Nº 403: Acute Inhalation Toxicity (12/05/1981)");
guidelineMapping.put(List.of("433", "2018"), "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)");
guidelineMapping.put(List.of("433", "2017"), "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)");
guidelineMapping.put(List.of("436", "2009"), "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)");
guidelineMapping.put(List.of("404", "1981"), "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)");
guidelineMapping.put(List.of("404", "1992"), "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)");
guidelineMapping.put(List.of("404", "2002"), "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)");
guidelineMapping.put(List.of("404", "2015"), "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)");
guidelineMapping.put(List.of("405", "2017"), "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)");
guidelineMapping.put(List.of("405", "2012"), "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)");
guidelineMapping.put(List.of("405", "2002"), "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)");
guidelineMapping.put(List.of("405", "1987"), "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)");
guidelineMapping.put(List.of("429", "2002"), "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)");
guidelineMapping.put(List.of("429", "2010"), "Nº 429: Skin Sensitisation (23/07/2010)");
guidelineMapping.put(List.of("442A", "2018"), "Nº 442A: Skin Sensitization (23/07/2018)");
guidelineMapping.put(List.of("442B", "2018"), "Nº 442B: Skin Sensitization (27/06/2018)");
guidelineMapping.put(List.of("471", "1997"), "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)");
guidelineMapping.put(List.of("471", "2020"), "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)");
guidelineMapping.put(List.of("406", "1992"), "Nº 406: Skin Sensitisation (1992)");
guidelineMapping.put(List.of("428", "2004"), "Nº 428: Split-Thickness Skin test (2004)");
guidelineMapping.put(List.of("438", "2018"), "Nº 438: Eye Irritation (26/06/2018)");
guidelineMapping.put(List.of("439", "2019"), "Nº 439: Skin Irritation (2019)");
guidelineMapping.put(List.of("474", "2016"), "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)");
guidelineMapping.put(List.of("487", "2016"), "Nº 487: Micronucleus Human Lymphocytes (2016)");
Map<String, List<List<String>>> mappings = guidelineMapping.entrySet()
.stream()
.collect(Collectors.groupingBy(Map.Entry::getValue, Collectors.mapping(Map.Entry::getKey, toList())));
int id = 1;
StringBuilder sb = new StringBuilder();
for (String guideline : mappings.keySet()) {
String year = getString(mappings.get(guideline).stream().map(l -> l.get(0)).distinct().toList());
String number = getString(mappings.get(guideline).stream().map(l -> l.get(1)).distinct().toList());
sb.append(stem.replaceAll(("klasjfguidelineklasjf"), guideline)
.replaceAll(("klasjfidklasjf"), String.valueOf(id))
.replaceAll(("klasjfnumberklasjf"), number)
.replaceAll(("klasjfyearklasjf"), year));
id++;
}
System.out.println(sb);
}
private static String getString(List<String> strings) {
if (strings.size() == 1) {
return String.format("value == \"%s\"", strings.get(0));
}
StringBuilder sb = new StringBuilder();
sb.append("(");
for (String string : strings) {
sb.append(String.format("value == \"%s\"", strings));
sb.append(" || ");
}
sb.delete(sb.length() - 4, sb.length());
sb.append(")");
return sb.toString();
}
@Test
@SneakyThrows
@Disabled
@ -232,6 +140,7 @@ class RuleFileFactoryTest {
System.out.println(result);
}
/**
* Manual test to quickly unescape a rules file. Just change the path to whatever rule file you want to unescape
* and it will generate an 'output.txt' with the result.
@ -262,6 +171,7 @@ class RuleFileFactoryTest {
unescapeRulesFile(pathToFile, "new.txt");
}
@SneakyThrows
private void unescapeRulesFile(String pathToFile, String outputName) {

15
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/translation/OldRulesParserTest.java
View File

@ -61,8 +61,8 @@ class OldRulesParserTest {
public void printTranslationsTest() {
List<OldRulesParser.OldRulesCsvRecord> records = OldRulesParser.getOldRulesCsvRecords(RuleManagementResources.getOldRulesCsvInputStream());
List.of(RuleType.SYN, RuleType.CBI, RuleType.PII, RuleType.ETC, RuleType.AI).forEach(type -> {
List<RuleUnit> rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type).ruleUnits();
List.of(RuleType.fromString("SYN"), RuleType.fromString("CBI"), RuleType.fromString("PII"), RuleType.fromString("ETC"), RuleType.fromString("AI")).forEach(type -> {
List<RuleUnit> rulesOfClass = RuleFileParser.buildBluePrintFromAllRulesFile(ApplicationType.RM).findRuleClassByType(type).orElseThrow().ruleUnits();
rulesOfClass.forEach(unit -> printOldRulesThatTranslatesToNewRule(unit, records));
});
}
@ -97,7 +97,6 @@ class OldRulesParserTest {
}
@Test
@Disabled
@SneakyThrows
@ -138,14 +137,12 @@ class OldRulesParserTest {
// String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/RED/dossier-templates-v2/";
// Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")), Files.walk(Path.of(dossierTemplatesRepo + "docu")), Files.walk(Path.of(dossierTemplatesRepo + "qa")))
String dossierTemplatesRepo = "/home/aoezyetimoglu/repositories/PROJECTMANAGEMENT/Syngenta/business-logic/";
Stream.of(
Files.walk(Path.of(dossierTemplatesRepo + "dev")),
Stream.of(Files.walk(Path.of(dossierTemplatesRepo + "dev")),
Files.walk(Path.of(dossierTemplatesRepo + "dev-v2")),
Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-eu-reg")),
Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")),
Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg")) //
)
.flatMap(Function.identity())//
Files.walk(Path.of(dossierTemplatesRepo + "prod-cp-global-reg")), Files.walk(Path.of(dossierTemplatesRepo + "prod-seeds-reg"))
//
).flatMap(Function.identity())//
.filter(path -> path.getFileName().toString().equals("rules.drl"))//
.map(Path::toFile)//
.forEach(this::translateOldRuleFile);

1545
redaction-service-v1/rules-management/src/test/resources/all_redact_manager_rules.drl
View File

File diff suppressed because it is too large Load Diff

1397
redaction-service-v1/rules-management/src/test/resources/all_rules_documine.drl
View File

File diff suppressed because it is too large Load Diff