Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-9774 - Update rules

Browse Source
This commit is contained in:
Corina Olariu 2024-10-25 07:06:44 +02:00
parent 45cd7f3d98
commit 569699139f
22 changed files with 2455 additions and 1453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/AnalysisFinalizationService.java
View File

@ -61,6 +61,7 @@ public class AnalysisFinalizationService {
// as workaround for duplicate key exceptions occurring due to simultaneous analyses and reanalyses save instead of insert is used
// also analysis numbers should be incremented in every follow-up request, so checking if the log exists is not needed
if (!redactionStorageService.entityLogExists(analyzeRequest.getDossierId(), analyzeRequest.getFileId())) {
entityLog.setEntityLogEntry(entityLogChanges.getNewEntityLogEntries());
redactionStorageService.saveEntityLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId(), entityLog);
} else {

5
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityChangeLogService.java
View File

@ -41,7 +41,7 @@ public class EntityChangeLogService {
var now = OffsetDateTime.now();
if (previousEntityLogEntries.isEmpty()) {
newEntityLogEntries.forEach(entry -> entry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap())));
return new EntryChanges(newEntityLogEntries, new ArrayList<>());
return new EntryChanges(newEntityLogEntries.stream().filter(entry -> !entry.getState().equals(EntryState.REMOVED)).collect(Collectors.toList()), new ArrayList<>());
}
List<EntityLogEntry> toInsert = new ArrayList<>();
@ -51,6 +51,9 @@ public class EntityChangeLogService {
.filter(entry -> entry.getId().equals(entityLogEntry.getId()))
.findAny();
if (optionalPreviousEntity.isEmpty()) {
if (entityLogEntry.getState().equals(EntryState.REMOVED)) {
continue;
}
entityLogEntry.getChanges().add(new Change(analysisNumber, ChangeType.ADDED, now, Collections.emptyMap()));
toInsert.add(entityLogEntry);
continue;

1
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/EntityLogCreatorService.java
View File

@ -180,7 +180,6 @@ public class EntityLogCreatorService {
.stream()
.filter(entity -> !entity.getValue().isEmpty())
.filter(EntityLogCreatorService::notFalsePositiveOrFalseRecommendationOrRemoval)
.filter(entity -> !entity.removed())
.toList();
List<Image> images = document.streamAllImages()

40
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -1454,12 +1454,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -1468,10 +1479,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1593,15 +1615,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -307,7 +307,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var desireeEtAl = findEntityByTypeAndValue(entityLog, "CBI_author", "Desiree").filter(e -> e.getEntryType().equals(EntryType.ENTITY))
.filter(e -> e.getMatchedRule().startsWith("CBI.7"))
.filter(e -> e.getMatchedRule().startsWith("CBI.0"))
.findAny()
.orElseThrow();
IdRemoval removal = buildIdRemoval(desireeEtAl.getId());

55
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -245,9 +245,9 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var values = redactionLog.getEntityLogEntry()
var values = entityLog.getEntityLogEntry()
.stream()
.map(EntityLogEntry::getValue)
.collect(Collectors.toList());
@ -1279,6 +1279,10 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
@Test
public void entityIsAppliedAfterRecategorize() throws IOException {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");
when(rulesClient.getRules(TEST_DOSSIER_TEMPLATE_ID, RuleFileType.ENTITY)).thenReturn(JSONPrimitive.of(EFSA_SANITISATION_RULES));
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
@ -1299,9 +1303,12 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
.findFirst()
.get();
var newId = UUID.randomUUID().toString();
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
// .annotationId(newId)
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
@ -1311,6 +1318,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
// .annotationId(newId)
.type("CBI_author")
.legalBasis("")
.requestDate(OffsetDateTime.now())
@ -1326,6 +1334,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
var changedAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
// .filter(entityLogEntry -> entityLogEntry.getId().equals(newId))
.findFirst()
.get();
@ -1736,14 +1745,14 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var listCBI_0_0 = entityLog.getEntityLogEntry()
var listCBI_0_3 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
.toList();
var listCBI_0_1 = entityLog.getEntityLogEntry()
var listCBI_0_4 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
.toList();
FileAttribute vertebrateNo = FileAttribute.builder().id("fileAttributeId").label("Vertebrate Study").placeholder("{fileattributes.vertebrateStudy}").value("No").build();
@ -1755,18 +1764,18 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
var entityLog2 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var list2CBI_0_0 = entityLog2.getEntityLogEntry()
var list2CBI_0_3 = entityLog2.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
.toList();
var list2CBI_0_1 = entityLog2.getEntityLogEntry()
var list2CBI_0_4 = entityLog2.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
.toList();
assertEquals(listCBI_0_0.size(), list2CBI_0_1.size());
assertEquals(listCBI_0_1.size(), list2CBI_0_0.size());
assertEquals(listCBI_0_3.size(), list2CBI_0_4.size());
assertEquals(listCBI_0_4.size(), list2CBI_0_3.size());
request.setFileAttributes(List.of(vertebrateYes));
@ -1776,18 +1785,18 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
var entityLog3 = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var list3CBI_0_0 = entityLog3.getEntityLogEntry()
var list3CBI_0_3 = entityLog3.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.0"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.3"))
.toList();
var list3CBI_0_1 = entityLog3.getEntityLogEntry()
var list3CBI_0_4 = entityLog3.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.1"))
.filter(entityLogEntry -> entityLogEntry.getMatchedRule().startsWith("CBI.0.4"))
.toList();
assertEquals(list3CBI_0_0.size(), list2CBI_0_1.size());
assertEquals(list3CBI_0_1.size(), list2CBI_0_0.size());
assertEquals(list3CBI_0_3.size(), list2CBI_0_4.size());
assertEquals(list3CBI_0_4.size(), list2CBI_0_3.size());
}
@ -1979,7 +1988,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
.get("reason"), "No vertebrate found -> removed by manual override");
assertEquals(responseDavidKsenia.getChanges()
.get(1).getPropertyChanges()
.get("matchedRule"), "CBI.3.2 -> ");
.get("matchedRule"), "CBI.13.2 -> ");
assertEquals(responseDavidKsenia.getChanges()
.get(1).getPropertyChanges()
.get("state"), "SKIPPED -> IGNORED");
@ -2033,7 +2042,7 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
.get("reason"), "No vertebrate found -> Recategorized entities are applied by default., recategorized by manual override");
assertEquals(responseDavidKsenia.getChanges()
.get(1).getPropertyChanges()
.get("matchedRule"), "CBI.3.2 -> MAN.3.3");
.get("matchedRule"), "CBI.13.2 -> MAN.3.3");
assertEquals(responseDavidKsenia.getChanges()
.get(1).getPropertyChanges()
.get("legalBasis"), " -> new legal basis");
@ -2171,11 +2180,13 @@ public class RedactionIntegrationTest extends RulesIntegrationTest {
.get();
var entityLogEntry2 = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getValue().equals(assessment))
.filter(entityLogEntry -> entityLogEntry.getId().equals("64795d65b8654503a25f7c44fce71fcb"))
.findFirst()
.get();
assertEquals(entityLogEntry1.getState(), EntryState.APPLIED);
assertEquals(entityLogEntry1.getState(), EntryState.REMOVED);
assertEquals(entityLogEntry1.getMatchedRule(), "DICT.0.0");
assertEquals(entityLogEntry2.getState(), EntryState.APPLIED);
assertEquals(entityLogEntry2.getMatchedRule(), "CBI.0.3");
assertThat(entityLogEntry2.getStartOffset() > entityLogEntry1.getEndOffset());
}

21
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/entity/TextEntityTest.java
View File

@ -16,11 +16,22 @@ public class TextEntityTest {
PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
entity.skip("CBI.1.0", "");
entity.skip("CBI.2.0", "");
entity.skip("CBI.3.0", "");
entity.skip("CBI.4.1", "");
entity.skip("CBI.4.0", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.4.1");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(4);
entity.skip("CBI.13.1", "");
entity.skip("CBI.13.3", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.3");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
}
@Test
public void testMatchedRule2() {
PrecursorEntity entity = PrecursorEntity.builder().type("PII").entityType(EntityType.ENTITY).build();
entity.redact("PII.4.1", "", "legal");
entity.apply("MAN.3.0", "");
entity.apply("MAN.3.3", "");
entity.skip("CBI.13.2", "");
assertThat(entity.getMatchedRule().getRuleIdentifier().toString()).isEqualTo("CBI.13.2");
assertThat(entity.getMatchedRuleUnit()).isEqualTo(13);
}

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/ManualChangesEnd2EndTest.java
View File

@ -322,7 +322,7 @@ public class ManualChangesEnd2EndTest extends AbstractRedactionIntegrationTest {
.findFirst()
.get();
assertEquals("CBI.7.0", asyaLyon.getMatchedRule());
assertEquals("CBI.0.0", asyaLyon.getMatchedRule());
assertEquals("Published Information found in section", asyaLyon.getReason());
assertNotEquals(asyaLyon.getState(), EntryState.APPLIED);

19
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/manualchanges/PrecursorEntityTest.java
View File

@ -18,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogChanges;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntityLogEntry;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle;
@ -134,15 +135,17 @@ public class PrecursorEntityTest extends BuildDocumentIntegrationTest {
assertEquals(1, notFoundManualEntities.size());
assertTrue(document.getEntities().isEmpty());
List<EntityLogEntry> redactionLogEntries = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
document,
notFoundManualEntities,
new DictionaryVersion(),
0L).getEntityLog().getEntityLogEntry();
EntityLogChanges entityLogChanges = entityLogCreatorService.createInitialEntityLog(new AnalyzeRequest(),
document,
notFoundManualEntities,
new DictionaryVersion(),
0L);
List<EntityLogEntry> entityLogEntries = entityLogChanges.getEntityLog().getEntityLogEntry();
assertEquals(1, entityLogEntries.size());
assertEquals(value, entityLogEntries.get(0).getValue());
assertEquals(type, entityLogEntries.get(0).getType());
assertEquals(1, redactionLogEntries.size());
assertEquals(value, redactionLogEntries.get(0).getValue());
assertEquals(type, redactionLogEntries.get(0).getType());
return new DocumentAndEntity(document, notFoundManualEntities.get(0));
}

273
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -85,20 +85,58 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.skip("CBI.0.0", "Author found by \"et al\" regex");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -130,58 +168,6 @@ rule "CBI.2.0: Do not redact genitive CBI Author"
end
// Rule unit: CBI.7
rule "CBI.7.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.3: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
dictionary.recommendEverywhere(entity);
});
end
// Rule unit: CBI.9
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
@ -287,7 +273,6 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
$table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
// Rule unit: CBI.16
rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
when
@ -366,9 +351,9 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
when
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
dictionary.recommendEverywhere(laboratoryEntity);
});
end
@ -418,7 +403,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -427,7 +412,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -439,7 +424,7 @@ rule "PII.0.0: Redact all PII"
when
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
$pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
end
rule "PII.0.1: Redact all PII (non vertebrate study)"
@ -524,7 +509,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx"
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.redact("PII.2.0", "Found by Phone and Fax Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -534,7 +519,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
$section: Section(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Ph.") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
@ -542,7 +526,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -552,7 +536,6 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
$section: Section(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Ph.") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
@ -560,7 +543,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -568,7 +551,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
// Rule unit: PII.3
rule "PII.3.0: Redact telephone numbers by RegEx"
when
$section: Section(matchesRegex("[+]\\d{1,}"))
$section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
@ -577,7 +560,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx"
rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
$section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -586,12 +569,30 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
$section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
.forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
.forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.4
rule "PII.4.0: Redact line after contact information keywords"
@ -600,8 +601,6 @@ rule "PII.4.0: Redact line after contact information keywords"
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
@ -648,62 +647,6 @@ rule "PII.4.1: Redact line after contact information keywords"
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.5
rule "PII.5.0: Redact line after contact information keywords reduced"
@ -803,12 +746,7 @@ rule "PII.7.0: Redact contact information if applicant is found"
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "Tel.:", "Tel:", "Telephone number:", "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section)
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -909,7 +847,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -923,8 +861,9 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
@ -1027,7 +966,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "ETC.3.3: Redact logos"
@ -1321,12 +1260,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -1335,10 +1285,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1459,29 +1420,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

760
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

File diff suppressed because it is too large Load Diff

40
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1388,12 +1388,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -1402,10 +1413,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1527,15 +1549,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

253
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -85,23 +85,60 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.skip("CBI.0.0", "Author found by \"et al\" regex");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "CBI.0.1: Redact CBI Authors (vertebrate Study)"
rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: CBI.1
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
@ -130,34 +167,6 @@ rule "CBI.2.0: Do not redact genitive CBI Author"
end
// Rule unit: CBI.7
rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
// Rule unit: CBI.9
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
@ -305,7 +314,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -314,7 +323,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -374,7 +383,6 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
$section: Section(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Ph.") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
@ -382,7 +390,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -392,7 +400,6 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
$section: Section(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Ph.") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
@ -400,7 +407,7 @@ rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
containsString("Fel") ||
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^m]|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -417,7 +424,7 @@ rule "PII.2.3: Redact phone numbers without indicators"
rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
$section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -426,13 +433,86 @@ rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
$section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
.forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
.forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.5
rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.6
rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.7
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
when
@ -510,8 +590,9 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
@ -519,6 +600,16 @@ rule "PII.10.0: Redact study director abbreviation"
end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.10.1", "Personal information found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
@ -593,7 +684,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -617,6 +708,22 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
retract($dossierRedaction);
end
// Rule unit: ETC.12
rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.skip("ETC.12.2", "Dossier dictionary entry found");
end
rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$dossierRedaction: TextEntity(type() == "dossier_redaction")
then
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end
//------------------------------------ AI rules ------------------------------------
@ -880,12 +987,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -894,10 +1012,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1018,29 +1147,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

148
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual changes rules ------------------------------------
//------------------------------------ Manual redaction rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -304,7 +304,90 @@ rule "MAN.4.1: Apply legal basis change"
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.3
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
// Rule unit: X.4
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
retract($recommendation);
end
// Rule unit: X.5
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
retract($recommendation);
end
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
@ -316,6 +399,30 @@ rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATI
end
// Rule unit: X.6
rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
salience 32
when
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
retract($lowerRank);
end
rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
salience 32
when
$outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
then
$inner.getIntersectingNodes().forEach(node -> update(node));
$inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
retract($inner);
end
// Rule unit: X.8
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
salience 257
@ -361,29 +468,54 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: Remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($duplicate);
end

468
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -85,151 +85,20 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.3
rule "CBI.3.0: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.3.0",
"Vertebrate found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$section.getEntitiesOfType("vertebrate")
);
});
end
rule "CBI.3.1: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.3.1",
"Vertebrate found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$table.getEntitiesOfTypeInSameRow("vertebrate", entity)
);
});
end
rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
end
rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
then
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found"));
end
// Rule unit: CBI.4
rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.skipWithReferences(
"CBI.4.0",
"Vertebrate but a no redaction indicator found",
$section.getEntitiesOfType("no_redaction_indicator")
);
});
end
rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.skipWithReferences(
"CBI.4.1",
"Vertebrate but a no redaction indicator found",
Stream.concat(
$table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
$table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
);
});
end
// Rule unit: CBI.5
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("redaction_indicator"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.5.0",
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
);
});
end
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
when
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.5.1",
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(),
$table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
);
});
end
// Rule unit: CBI.7
rule "CBI.7.0: Add CBI_author with \"et al.\" RegEx"
// Rule unit: CBI.0
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.skip("CBI.0.0", "Author found by \"et al\" regex");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -237,12 +106,12 @@ rule "CBI.7.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -250,56 +119,11 @@ rule "CBI.7.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.7.3: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.7.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
dictionary.recommendEverywhere(entity);
});
end
// Rule unit: CBI.8
rule "CBI.8.0: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.8.0",
"must_redact entity found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$section.getEntitiesOfType("must_redact")
);
});
end
rule "CBI.8.1: Redacted because table row contains must_redact entity"
when
$table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.8.1",
"Must_redact found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$table.getEntitiesOfTypeInSameRow("must_redact", entity)
);
});
end
// Rule unit: CBI.9
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
@ -353,19 +177,19 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end
// Rule unit: CBI.12
// Rule unit: CBI.12 - table rules remains
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
$rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
$rowCell: TableCell(!header, containsAnyString("Yes", "Y", "yes"), !containsAnyString("Y/N"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
then
entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT)
entityCreationService.bySemanticNode($rowCell, "redaction_indicator", EntityType.HINT)
.ifPresent(yesEntity -> {
yesEntity.skip("CBI.12.0", "must_redact");
yesEntity.skip("CBI.12.0", "redaction_indicator");
});
end
@ -376,7 +200,7 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
TableCell(!header, containsAnyString("Yes", "Y", "yes"), !containsAnyString("Y/N"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
$authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
then
entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
@ -384,7 +208,6 @@ rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s
authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
});
end
rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
@ -419,6 +242,126 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.12.4",
"Redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$table.getEntitiesOfTypeInSameRow("redaction_indicator", entity)
);
});
end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
then
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.12.5",
"Vertebrate found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$table.getEntitiesOfTypeInSameRow("vertebrate", entity)
);
});
end
// Rule unit: CBI.13 - section rules
// from CBI.3.0
rule "CBI.13.1: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.13.1",
"Vertebrate found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
$section.getEntitiesOfType("vertebrate")
);
});
end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.skipWithReferences(
"CBI.13.3",
"Vertebrate but a no redaction indicator found",
$section.getEntitiesOfType("no_redaction_indicator")
);
});
end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("redaction_indicator"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.13.4",
"no_redaction_indicator but also redaction_indicator found",
"Reg (EC) No 1107/2009 Art. 63 (2g)",
Stream.concat(
$section.getEntitiesOfType("redaction_indicator").stream(),
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
);
});
end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.applyWithReferences(
"CBI.13.5",
"must_redact entity found",
"Reg (EC) No 1107/2009 Art. 63 (2b)",
$section.getEntitiesOfType("must_redact")
);
});
end
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
@ -450,7 +393,8 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
then
entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section)
.forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
entityCreationService.byString($residueKeyword, "must_redact", EntityType.HINT, $section)
.forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -471,7 +415,8 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
then
entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table)
.forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
entityCreationService.byString($residueKeyword, "must_redact", EntityType.HINT, $table)
.forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
@ -536,9 +481,9 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
when
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
dictionary.recommendEverywhere(laboratoryEntity);
});
end
@ -588,7 +533,7 @@ rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (no
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -597,7 +542,7 @@ rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ve
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -609,7 +554,7 @@ rule "PII.0.0: Redact all PII"
when
$pii: TextEntity(type() == "PII", dictionaryEntry)
then
$pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
$pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
end
rule "PII.0.1: Redact all PII (non vertebrate study)"
@ -687,8 +632,6 @@ rule "PII.4.0: Redact line after contact information keywords"
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
@ -735,62 +678,6 @@ rule "PII.4.1: Redact line after contact information keywords"
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
end
rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.6
rule "PII.6.0: Redact line between contact keywords"
@ -850,12 +737,7 @@ rule "PII.7.0: Redact contact information if applicant is found"
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "Tel.:", "Tel:", "Telephone number:", "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section)
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -956,7 +838,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
then
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document, 200)
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -1064,7 +946,7 @@ rule "ETC.3.2: Redact logos (vertebrate study)"
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$logo: Image(imageType == ImageType.LOGO)
then
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
rule "ETC.3.3: Redact logos"
@ -1251,7 +1133,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual changes rules ------------------------------------
//------------------------------------ Manual redaction rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1410,12 +1292,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -1424,10 +1317,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1548,29 +1452,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

98
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -72,12 +72,46 @@ query "getFileAttributes"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Redact CBI Authors (non vertebrate Study)"
rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.redact("CBI.0.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.recommendEverywhere(entity);
});
end
rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.0", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
$entity.redact("CBI.0.3", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
then
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -332,12 +366,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -346,10 +391,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -470,29 +526,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

60
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -458,12 +458,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -472,10 +483,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -607,29 +629,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

82
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -358,12 +358,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -372,14 +383,49 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.3
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
@ -507,29 +553,41 @@ rule "X.10.0: remove false positives of ai"
// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
$nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
then
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
retract($dictionaryEntity);
$nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when
$manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
$nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
then
$manualEntity.addEngines($nonManualEntity.getEngines());
$nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
retract($nonManualEntity);
end
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

772
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

File diff suppressed because it is too large Load Diff

766
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

File diff suppressed because it is too large Load Diff

40
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1454,12 +1454,23 @@ rule "MAN.4.1: Apply legal basis change"
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges(), !removed())
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
salience 65
when
$larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
then
$contained.getIntersectingNodes().forEach(node -> update(node));
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
update($contained);
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
retract($contained);
end
@ -1468,10 +1479,21 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1593,15 +1615,15 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
//------------------------------------ Dictionary merging rules ------------------------------------
// Rule unit: DICT.0
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
salience 64
when
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
not TextEntity(containedBy($entity), engines contains Engine.DOSSIER_DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
end

2
redaction-service-v1/rules-management/src/test/java/com/knecon/fforesight/utility/rules/management/factory/RuleFileFactoryTest.java
View File

@ -97,7 +97,7 @@ class RuleFileFactoryTest {
void createFileFromIdentifiersForRedactionService() {
// This is exactly the string used for the current rules.drl in the redaction-service
String identifiers = "CBI.5.*, CBI.9.*, CBI.11.*, AI.1.*, PII.4.*, ETC.8.*, PII.0.*, ETC.6.*, SYN.0.*, CBI.3.*, ETC.4.*, ETC.3.*, PII.12.*, ETC.1.*, PII.9.*, PII.7.*, CBI.12.*, X.*.*, CBI.14.*, CBI.16.*, CBI.18.*, CBI.4.*, AI.0.*, CBI.8.*, PII.1.*, ETC.7.*, LDS.*.*, MAN.*.*, ETC.5.*, PII.11.*, ETC.2.*, CBI.20.*, FA.*.*, PII.8.*, PII.6.*, CBI.15.*, CBI.17.*, CBI.19.*";
String identifiers = "CBI.9.*, CBI.11.*, AI.1.*, PII.4.*, ETC.8.*, PII.0.*, ETC.6.*, SYN.0.*, ETC.4.*, ETC.3.*, PII.12.*, ETC.1.*, PII.9.*, PII.7.*, CBI.12.*, X.*.*, CBI.14.*, CBI.16.*, CBI.18.*, AI.0.*, PII.1.*, ETC.7.*, LDS.*.*, MAN.*.*, ETC.5.*, PII.11.*, ETC.2.*, CBI.20.*, FA.*.*, PII.8.*, PII.6.*, CBI.15.*, CBI.17.*, CBI.19.*";
String result = RuleFileFactory.createFileFromIdentifiers(identifiers, ApplicationType.RM);
System.out.println(result);
try (var out = new FileOutputStream(