Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-6009' into 'master'

Browse Source 
RED-6009: Document Tree Structure

Closes RED-6009

See merge request redactmanager/redaction-service!10
This commit is contained in:
Kilian Schüttler 2023-06-21 13:21:06 +02:00
commit 847a50c32a
11 changed files with 1444 additions and 850 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/pom.xml
View File

@ -12,7 +12,7 @@
<artifactId>redaction-service-api-v1</artifactId>
<properties>
<persistence-service.version>2.75.0</persistence-service.version>
<persistence-service.version>2.79.0</persistence-service.version>
</properties>
<dependencies>

20
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
View File

@ -49,7 +49,7 @@ public class RedactionEntity {
Set<Engine> engines;
Set<RedactionEntity> references;
@Builder.Default
Deque<Integer> matchedRules = new LinkedList<>();
Deque<String> matchedRules = new LinkedList<>();
String redactionReason;
String legalBasis;
@ -120,16 +120,26 @@ public class RedactionEntity {
}
public void addMatchedRule(int ruleNumber) {
public void addMatchedRule(String ruleIdentifier) {
matchedRules.add(ruleNumber);
matchedRules.add(ruleIdentifier);
}
public int getMatchedRule() {
public int getMatchedRuleUnit() {
String[] values = getMatchedRule().split("\\.");
if (values.length < 2) {
return -1;
}
return Integer.parseInt(values[1]);
}
public String getMatchedRule() {
if (matchedRules.isEmpty()) {
return 0;
return "";
}
return matchedRules.getLast();
}

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java
View File

@ -42,7 +42,7 @@ public class Image implements GenericSemanticNode {
@Builder.Default
String legalBasis = "";
@Builder.Default
int matchedRule = -1;
String matchedRule = "";
@EqualsAndHashCode.Exclude
Page page;

9
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
View File

@ -184,11 +184,12 @@ public interface SemanticNode {
/**
* Checks whether this SemanticNode has any Entity of the provided type.
* Ignores Entity with ignored == true or removed == true.
*
* @param type string representing the type of entity to check for
* @return true, if this SemanticNode has at least one Entity of the provided type
*/
default boolean hasActiveEntitiesOfType(String type) {
default boolean hasEntitiesOfType(String type) {
return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> redactionEntity.getType().equals(type));
}
@ -196,11 +197,12 @@ public interface SemanticNode {
/**
* Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
* Ignores Entity with ignored == true or removed == true.
*
* @param type string representing the type of entities to return
* @return List of RedactionEntities of any the type
*/
default List<RedactionEntity> getActiveEntitiesOfType(String type) {
default List<RedactionEntity> getEntitiesOfType(String type) {
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.getType().equals(type)).toList();
}
@ -208,11 +210,12 @@ public interface SemanticNode {
/**
* Returns a List of Entities in this SemanticNode which have any of the provided types such as "CBI_author".
* Ignores Entity with ignored == true or removed == true.
*
* @param types A list of strings representing the types of entities to return
* @return List of RedactionEntities of any provided type
*/
default List<RedactionEntity> getActiveEntitiesOfType(List<String> types) {
default List<RedactionEntity> getEntitiesOfType(List<String> types) {
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(types)).toList();
}

11
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Table.java
View File

@ -103,11 +103,12 @@ public class Table implements SemanticNode {
/**
* Streams all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
* Ignores Entity with ignored == true or removed == true.
*
* @param types type strings to check whether a row contains an entity like them
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
*/
public Stream<RedactionEntity> streamEntitiesWhereRowContainsActiveEntitiesOfType(List<String> types) {
public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
List<Integer> rowsWithEntityOfType = getEntities().stream()
.filter(RedactionEntity::isActive)
@ -124,11 +125,12 @@ public class Table implements SemanticNode {
/**
* Streams all entities in this table, that appear in a row, which contains no entity of any of the provided types.
* Ignores Entity with ignored == true or removed == true.
*
* @param types type strings to check whether a row contains an entity like them
* @return Stream of all entities in this table, that appear in a row, which contains at least one entity with any of the provided types.
*/
public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List<String> types) {
public Stream<RedactionEntity> streamEntitiesWhereRowContainsNoEntitiesOfType(List<String> types) {
return IntStream.range(0, numberOfRows)
.boxed()
@ -272,19 +274,20 @@ public class Table implements SemanticNode {
/**
* Finds all entities of the provided type, which appear in the same row that the provided entity appears in.
* Ignores Entity with ignored == true or removed == true.
*
* @param type the type of entities to search for
* @param redactionEntity the entity, which appears in the row to search
* @return List of all entities of the provided type, which appear in the same row that the provided entity appears in.
*/
public List<RedactionEntity> getActiveEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
public List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
return redactionEntity.getIntersectingNodes()
.stream()
.filter(node -> node instanceof TableCell)
.map(node -> (TableCell) node)
.flatMap(tableCellNode -> streamRow(tableCellNode.getRow()))
.map(cell -> cell.getActiveEntitiesOfType(type))
.map(cell -> cell.getEntitiesOfType(type))
.flatMap(Collection::stream)
.toList();
}

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -1,22 +1,5 @@
package com.iqser.red.service.redaction.v1.server.layoutparsing.document.services;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedEndByRegex;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedStartByRegex;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.DocumentTree;
@ -31,9 +14,17 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.Re
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import com.iqser.red.service.redaction.v1.server.redaction.utils.IdBuilder;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedEndByRegex;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.getExpandedStartByRegex;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
@Slf4j
@Service
@ -252,7 +243,7 @@ public class EntityCreationService {
mergedEntity.addEngines(entitiesToMerge.stream().flatMap(entityNode -> entityNode.getEngines().stream()).collect(Collectors.toSet()));
entitiesToMerge.stream().map(RedactionEntity::getMatchedRules).flatMap(Collection::stream).forEach(mergedEntity::addMatchedRule);
RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRule)).orElse(entitiesToMerge.get(0));
RedactionEntity entityWithHigherRuleNumber = entitiesToMerge.stream().max(Comparator.comparingInt(RedactionEntity::getMatchedRuleUnit)).orElse(entitiesToMerge.get(0));
mergedEntity.setRedactionReason(entityWithHigherRuleNumber.getRedactionReason());
mergedEntity.setLegalBasis(entityWithHigherRuleNumber.getLegalBasis());
mergedEntity.setDictionaryEntry(entitiesToMerge.stream().anyMatch(RedactionEntity::isDictionaryEntry));

1268
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl Normal file
View File

File diff suppressed because it is too large Load Diff

204
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -62,7 +62,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
).forEach(entity -> {
entity.setRedactionReason("hint_only");
entity.addMatchedRule(0);
entity.addMatchedRule("SYN.0.0");
entity.addEngine(Engine.RULE);
insert(entity);
});
@ -74,59 +74,59 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
// Rule unit: CBI.3
rule "CBI.3.0: Redacted because Section contains Vertebrate"
when
$section: Section(!hasTables(), hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addReferences($section.getActiveEntitiesOfType("vertebrate"));
entity.addMatchedRule("CBI.3.0");
entity.addReferences($section.getEntitiesOfType("vertebrate"));
});
end
rule "CBI.3.1: Redacted because Table Row contains Vertebrate"
when
$table: Table(hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate"))
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addMatchedRule("CBI.3.1");
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
});
end
rule "CBI.3.2: Don't redact because Section doesn't contain Vertebrate"
when
$section: Section(!hasTables(), !hasActiveEntitiesOfType("vertebrate"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("No vertebrate found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addMatchedRule("CBI.3.2");
});
end
rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
when
$table: Table(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address"))
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
then
$table.streamEntitiesWhereRowContainsNoActiveEntitiesOfType(List.of("vertebrate"))
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("No vertebrate found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(3);
entity.addMatchedRule("CBI.3.3");
});
end
@ -135,35 +135,35 @@ rule "CBI.3.3: Dont redact because Table Row doesn't contain Vertebrate"
rule "CBI.4.0: Dont redact Names and Addresses if no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate but a no redaction indicator found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(4);
entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator"));
entity.addMatchedRule("CBI.4.0");
entity.addReferences($section.getEntitiesOfType("no_redaction_indicator"));
});
end
rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is found in Table Row"
when
$table: Table(hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("Vertebrate but a no redaction indicator found");
entity.setRedaction(false);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(4);
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
entity.addMatchedRule("CBI.4.1");
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
});
end
@ -172,38 +172,38 @@ rule "CBI.4.1: Dont redact Names and Addresses if no_redaction_indicator is foun
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in section"
when
$section: Section(!hasTables(),
hasActiveEntitiesOfType("redaction_indicator"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
hasEntitiesOfType("redaction_indicator"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(5);
entity.addReferences($section.getActiveEntitiesOfType("no_redaction_indicator"));
entity.addReferences($section.getActiveEntitiesOfType("redaction_indicator"));
entity.addMatchedRule("CBI.5.0");
entity.addReferences($section.getEntitiesOfType("no_redaction_indicator"));
entity.addReferences($section.getEntitiesOfType("redaction_indicator"));
});
end
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Table Row"
when
$table: Table(hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator"))
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no-redaction_indicator"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("no_redaction_indicator but also redaction_indicator found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(5);
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
entity.addMatchedRule("CBI.5.1");
entity.addReferences($table.getEntitiesOfTypeInSameRow("vertebrate", entity));
entity.addReferences($table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity));
});
end
@ -211,32 +211,32 @@ rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also red
// Rule unit: CBI.8
rule "CBI.8.0: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("must_redact entity found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(15);
entity.addReferences($section.getActiveEntitiesOfType("must_redact"));
entity.addMatchedRule("CBI.8.0");
entity.addReferences($section.getEntitiesOfType("must_redact"));
});
end
rule "CBI.8.1: Redacted because Table Row contains must_redact entity"
when
$table: Table(hasActiveEntitiesOfType("must_redact"), (hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$table.streamEntitiesWhereRowContainsActiveEntitiesOfType(List.of("must_redact"))
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
.forEach(entity -> {
entity.setRedactionReason("must_redact entity found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.setRedaction(true);
entity.addEngine(Engine.RULE);
entity.addMatchedRule(15);
entity.addReferences($table.getActiveEntitiesOfTypeInSameRow("must_redact", entity));
entity.addMatchedRule("CBI.8.1");
entity.addReferences($table.getEntitiesOfTypeInSameRow("must_redact", entity));
});
end
@ -254,7 +254,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(9);
redactionEntity.addMatchedRule("CBI.9.0");
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author(s) found");
redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
@ -274,7 +274,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(9);
redactionEntity.addMatchedRule("CBI.9.1");
redactionEntity.addEngine(Engine.RULE);
redactionEntity.setRedactionReason("Author found");
redactionEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
@ -290,7 +290,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
when
$table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
then
$table.getActiveEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
$table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
@ -308,7 +308,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.addMatchedRule(12);
redactionEntity.addMatchedRule("CBI.12.0");
redactionEntity.setRedactionReason("Author(s) header found");
redactionEntity.addEngine(Engine.RULE);
insert(redactionEntity);
@ -324,7 +324,7 @@ rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \
.forEach(authorEntity -> {
authorEntity.setRedaction(false);
authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study");
authorEntity.addMatchedRule(12);
authorEntity.addMatchedRule("CBI.12.1");
});
end
@ -338,7 +338,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert
authorEntity.setRedaction(true);
authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
authorEntity.addMatchedRule(12);
authorEntity.addMatchedRule("CBI.12.2");
});
end
@ -351,7 +351,7 @@ rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at
$sponsorEntity.setRedaction(true);
$sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company");
$sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
$sponsorEntity.addMatchedRule(14);
$sponsorEntity.addMatchedRule("CBI.14.0");
end
@ -375,10 +375,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
.forEach(keywordEntity -> insert(keywordEntity));
$section.getActiveEntitiesOfType(List.of($keyword, $residueKeyword))
$section.getEntitiesOfType(List.of($keyword, $residueKeyword))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(15);
redactionEntity.addMatchedRule("CBI.15.0");
redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
@ -405,7 +405,7 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(15);
redactionEntity.addMatchedRule("CBI.15.1");
redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
@ -424,7 +424,7 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addMatchedRule(18);
entity.addMatchedRule("CBI.16.0");
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
@ -442,7 +442,7 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addMatchedRule(19);
entity.addMatchedRule("CBI.16.1");
entity.addEngine(Engine.RULE);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
@ -459,7 +459,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
.forEach(entity -> {
entity.setRedactionReason("Line after \"Source\" in Test Organism Section");
entity.addEngine(Engine.RULE);
entity.addMatchedRule(20);
entity.addMatchedRule("CBI.17.0");
insert(entity);
});
end
@ -472,7 +472,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
.forEach(entity -> {
entity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
entity.addEngine(Engine.RULE);
entity.addMatchedRule(20);
entity.addMatchedRule("CBI.17.1");
insert(entity);
});
end
@ -489,7 +489,7 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
)
then
RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
expandedEntity.addMatchedRule(0);
expandedEntity.addMatchedRule("CBI.18.0");
$entityToExpand.removeFromGraph();
retract($entityToExpand);
insert(expandedEntity);
@ -502,7 +502,7 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
$entityToExpand: RedactionEntity(type == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
expandedEntity.addMatchedRule(0);
expandedEntity.addMatchedRule("CBI.19.0");
insert(expandedEntity);
end
@ -517,7 +517,7 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(false);
laboratoryEntity.addMatchedRule(20);
laboratoryEntity.addMatchedRule("CBI.20.0");
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
@ -534,7 +534,7 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
.forEach(laboratoryEntity -> {
laboratoryEntity.setRedaction(true);
laboratoryEntity.addMatchedRule(20);
laboratoryEntity.addMatchedRule("CBI.20.1");
laboratoryEntity.addEngine(Engine.RULE);
laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found");
laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
@ -555,7 +555,7 @@ rule "PII.0.0: Redact all PII (non vertebrate study)"
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
$pii.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
$pii.addMatchedRule(0);
$pii.addMatchedRule("PII.0.0");
end
rule "PII.0.1: Redact all PII (vertebrate study)"
@ -566,7 +566,7 @@ rule "PII.0.1: Redact all PII (vertebrate study)"
$pii.setRedaction(true);
$pii.setRedactionReason("Personal Information found");
$pii.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
$pii.addMatchedRule(0);
$pii.addMatchedRule("PII.0.1");
end
@ -582,7 +582,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
emailEntity.addEngine(Engine.RULE);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
emailEntity.addMatchedRule(1);
emailEntity.addMatchedRule("PII.1.0");
insert(emailEntity);
});
end
@ -598,7 +598,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
emailEntity.addEngine(Engine.RULE);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
emailEntity.addMatchedRule(1);
emailEntity.addMatchedRule("PII.1.1");
insert(emailEntity);
});
end
@ -632,7 +632,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(4);
contactEntity.addMatchedRule("PII.4.0");
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
@ -667,7 +667,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(4);
contactEntity.addMatchedRule("PII.4.1");
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
@ -688,7 +688,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(6);
contactEntity.addMatchedRule("PII.6.0");
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
@ -707,7 +707,7 @@ rule "PII.6.1: redact line between contact keywords"
)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(6);
contactEntity.addMatchedRule("PII.6.1");
contactEntity.addEngine(Engine.RULE);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
@ -736,7 +736,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
entity.setRedaction(true);
entity.setRedactionReason("Applicant information was found");
entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
entity.addMatchedRule(7);
entity.addMatchedRule("PII.7.0");
entity.addEngine(Engine.RULE);
insert(entity);
});
@ -761,7 +761,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
entity.setRedaction(true);
entity.setRedactionReason("Applicant information was found");
entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addMatchedRule(7);
entity.addMatchedRule("PII.7.1");
entity.addEngine(Engine.RULE);
insert(entity);
});
@ -788,7 +788,7 @@ rule "PII.8.0: Redact contact information if producer is found"
entity.setRedaction(true);
entity.setRedactionReason("Producer was found");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
entity.addMatchedRule(8);
entity.addMatchedRule("PII.8.0");
entity.addEngine(Engine.RULE);
insert(entity);
});
@ -813,7 +813,7 @@ rule "PII.8.1: Redact contact information if producer is found"
entity.setRedaction(true);
entity.setRedactionReason("Producer was found");
entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
entity.addMatchedRule(8);
entity.addMatchedRule("PII.8.1");
entity.addEngine(Engine.RULE);
insert(entity);
});
@ -829,7 +829,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(9);
authorEntity.addMatchedRule("PII.9.0");
authorEntity.addEngine(Engine.RULE);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
@ -845,7 +845,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(9);
authorEntity.addMatchedRule("PII.9.1");
authorEntity.addEngine(Engine.RULE);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
@ -861,7 +861,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(9);
authorEntity.addMatchedRule("PII.9.2");
authorEntity.addEngine(Engine.RULE);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
@ -877,7 +877,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(9);
authorEntity.addMatchedRule("PII.9.3");
authorEntity.addEngine(Engine.RULE);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
@ -886,13 +886,29 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
end
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule("PII.11.0");
authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
authorEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
end
// Rule unit: PII.12
rule "PII.12.0: Expand PII entities with salutation prefix"
when
$entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
expandedEntity.addMatchedRule(12);
expandedEntity.addMatchedRule("PII.12.0");
insert(expandedEntity);
end
@ -906,7 +922,7 @@ rule "ETC.1.0: Redact Purity"
then
entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.addMatchedRule(1);
entity.addMatchedRule("ETC.1.0");
entity.addEngine(Engine.RULE);
entity.setRedaction(true);
entity.setRedactionReason("Purity found");
@ -922,7 +938,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(2);
$signature.setMatchedRule("ETC.2.0");
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -933,7 +949,7 @@ rule "ETC.2.0: Redact signatures (vertebrate study)"
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(2);
$signature.setMatchedRule("ETC.2.0");
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -946,7 +962,7 @@ rule "ETC.3.0: Redact logos (vertebrate study)"
$logo: Image(imageType == ImageType.LOGO)
then
$logo.setRedaction(true);
$logo.setMatchedRule(3);
$logo.setMatchedRule("ETC.3.0");
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -957,7 +973,7 @@ rule "ETC.3.1: Redact logos (non vertebrate study)"
$logo: Image(imageType == ImageType.LOGO)
then
$logo.setRedaction(true);
$logo.setMatchedRule(3);
$logo.setMatchedRule("ETC.3.1");
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end
@ -969,7 +985,7 @@ rule "ETC.4.0: Redact dossier dictionary entries"
$dossierRedaction: RedactionEntity(type == "dossier_redaction")
then
$dossierRedaction.setRedaction(true);
$dossierRedaction.addMatchedRule(4);
$dossierRedaction.addMatchedRule("ETC.4.0");
$dossierRedaction.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
$dossierRedaction.setRedactionReason("Specification of impurity found");
end
@ -997,7 +1013,7 @@ rule "ETC.6.0: Redact CAS Number"
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(101);
redactionEntity.addMatchedRule("ETC.6.0");
redactionEntity.setRedactionReason("Sample # found in Header");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
insert(redactionEntity);
@ -1024,7 +1040,7 @@ rule "ETC.8.0: Redact formulas (vertebrate study)"
$logo: Image(imageType == ImageType.FORMULA)
then
$logo.setRedaction(true);
$logo.setMatchedRule(3);
$logo.setMatchedRule("ETC.8.0");
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
end
@ -1035,7 +1051,7 @@ rule "ETC.8.1: Redact formulas (non vertebrate study)"
$logo: Image(imageType == ImageType.FORMULA)
then
$logo.setRedaction(true);
$logo.setMatchedRule(3);
$logo.setMatchedRule("ETC.8.1");
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
end

697
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_backup.drl
View File

@ -1,697 +0,0 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.ImageType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
global Document document
global EntityCreationService entityCreationService
global ManualRedactionApplicationService manualRedactionApplicationService
global NerEntitiesAdapter nerEntitiesAdapter
global Dictionary dictionary
// --------------------------------------- queries -------------------------------------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
// --------------------------------------- CBI rules -------------------------------------------------------------------
rule "0: Expand CBI_author entities with firstname initials"
no-loop true
when
$entityToExpand: RedactionEntity(type == "CBI_author",
value.matches("[^\\s]+"),
textAfter.startsWith(" "),
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
)
then
RedactionEntity expandedEntity = entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)");
expandedEntity.addMatchedRule(0);
$entityToExpand.removeFromGraph();
retract($entityToExpand);
insert(expandedEntity);
end
rule "0: Expand CBI_author and PII entities with salutation prefix"
when
$entityToExpand: RedactionEntity((type == "CBI_author" || type == "PII"), anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
RedactionEntity expandedEntity = entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*");
expandedEntity.addMatchedRule(0);
insert(expandedEntity);
end
rule "1: Redacted because Section contains Vertebrate"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(1);
redactionEntity.setRedactionReason("Vertebrate Found in this section");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
end
rule "2: Not Redacted because Section contains no Vertebrate"
when
$section: Section(!hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(2);
redactionEntity.setRedactionReason("No Vertebrate Found in this section");
});
end
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(3);
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator found in this section");
});
end
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(4);
redactionEntity.setRedactionReason("Vertebrate and a no-redaction-indicator, but also redaction-indicator, found in this section");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
});
end
rule "5: Do not redact Names and Addresses if published information found"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("published_information"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getActiveEntitiesOfType("published_information");
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
redactionEntity.addReferences(publishedInformationEntities);
});
end
rule "6.0: Add all Cell's with Header Author(s) as CBI_author"
when
$table: Table(hasHeader("Author(s)"))
then
$table.streamTableCellsWithHeader("Author(s)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.addMatchedRule(6);
redactionEntity.setRedactionReason("Author(s) header found");
insert(redactionEntity);
});
end
rule "6.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
when
$table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "N") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "No"))
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.setRedaction(false);
authorEntity.setRedactionReason("Not redacted because it's row does not belong to a vertebrate study");
authorEntity.setLegalBasis("");
authorEntity.addMatchedRule(6);
});
end
rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
when
$table: Table(hasRowWithHeaderAndValue("Vertebrate study Y/N", "Y") || hasRowWithHeaderAndValue("Vertebrate study Y/N", "Yes"))
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.setRedactionReason("Redacted because it's row belongs to a vertebrate study");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
authorEntity.addMatchedRule(7);
});
end
rule "8: Redact if must_redact entity is found"
when
$section: Section(hasActiveEntitiesOfType("must_redact"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("must_redact entry was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(8);
});
end
rule "9: Redact CBI_sponsor entities if preceded by \" batches produced at\""
when
$sponsorEntity: RedactionEntity(type == "CBI_sponsor", textBefore.contains("batches produced at"))
then
$sponsorEntity.setRedaction(true);
$sponsorEntity.setRedactionReason("Redacted because it represents a sponsor company");
$sponsorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
$sponsorEntity.addMatchedRule(9);
end
rule "10: Redact row if row contains \"determination of residues\" and livestock keyword"
when
$keyword: String() from List.of("livestock",
"live stock",
"tissue",
"tissues",
"liver",
"muscle",
"bovine",
"ruminant",
"ruminants")
$residueKeyword: String() from List.of("determination of residues", "determination of total residues")
$table: Table(containsStringIgnoreCase($residueKeyword)
&& containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
.forEach(keywordEntity -> insert(keywordEntity));
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Determination of residues and keyword \"" + $keyword + "\" was found.");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(10);
});
end
rule "11: Redact if CTL/* or BL/* was found"
when
$section: Section(excludesTables, (containsString("CTL/") || containsString("BL/")))
then
entityCreationService.byString("CTL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
redactionEntity.addMatchedRule(11);
});
end
rule "12: Add CBI_author with \"et al.\" Regex"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section(containsString("et al."))
then
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
.forEach(entity -> {
entity.setRedaction(true);
entity.setRedactionReason("Author found by \"et al\" regex");
entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
entity.addMatchedRule(12);
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
rule "13: Add recommendation for Addresses in Test Organism sections"
when
$section: Section(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(redactionEntity -> {
redactionEntity.setRedactionReason("Line after \"Source\" in Test Organism Section");
redactionEntity.addMatchedRule(13);
insert(redactionEntity);
});
end
rule "14: Add recommendation for Addresses in Test Animals sections"
when
$section: Section(excludesTables, containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(redactionEntity -> {
redactionEntity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
redactionEntity.addMatchedRule(14);
insert(redactionEntity);
});
end
// --------------------------------------- PII rules -------------------------------------------------------------------
rule "15: Redact all PII"
when
$pii: RedactionEntity(type == "PII", redaction == false)
then
$pii.setRedaction(true);
$pii.setRedactionReason("PII found");
$pii.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
$pii.addMatchedRule(15);
end
rule "16: Redact Emails by RegEx (Non vertebrate study)"
when
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section)
.forEach(emailEntity -> {
emailEntity.setRedaction(true);
emailEntity.setRedactionReason("Found by Email Regex");
emailEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
emailEntity.addMatchedRule(16);
insert(emailEntity);
});
end
rule "17: Redact line after contact information keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(17);
contactEntity.setRedactionReason("Found after \"" + $contactKeyword + "\" contact keyword");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
});
end
rule "18: redact line between contact keywords"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.setRedaction(true);
contactEntity.addMatchedRule(18);
contactEntity.setRedactionReason("Found between contact keywords");
contactEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
dictionary.addLocalDictionaryEntry("PII", contactEntity.getValue(), false);
});
end
rule "19: Redact AUTHOR(S)"
when
FileAttribute(placeholder == "{fileattributes.vertebrateStudy}", value == "true")
$section: Section(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(19);
authorEntity.setRedactionReason("AUTHOR(S) was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "20: Redact PERFORMING LABORATORY"
when
$section: Section(excludesTables, containsString("PERFORMING LABORATORY:"))
then
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(20);
authorEntity.setRedactionReason("PERFORMING LABORATORY was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "21: Redact On behalf of Sequani Ltd.:"
when
$section: Section(excludesTables, containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(21);
authorEntity.setRedactionReason("On behalf of Sequani Ltd.: Name Title was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "22: Redact On behalf of Syngenta Ltd.:"
when
$section: Section(excludesTables, containsString("On behalf of Syngenta Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.setRedaction(true);
authorEntity.addMatchedRule(21);
authorEntity.setRedactionReason("On behalf of Syngenta Ltd.: Name Title was found");
authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(authorEntity);
});
end
rule "26: Redact signatures"
when
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.setRedaction(true);
$signature.setMatchedRule(26);
$signature.setRedactionReason("Signature Found");
$signature.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "27: Redact formulas"
when
$formula: Image(imageType == ImageType.FORMULA)
then
$formula.setRedaction(true);
$formula.setMatchedRule(27);
$formula.setRedactionReason("Formula Found");
$formula.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "28: Redact logos"
when
$logo: Image(imageType == ImageType.LOGO)
then
$logo.setRedaction(true);
$logo.setMatchedRule(28);
$logo.setRedactionReason("Logo Found");
$logo.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
end
rule "29: Redact Dossier Redactions"
when
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
then
$dossierRedaction.setRedaction(true);
$dossierRedaction.addMatchedRule(29);
$dossierRedaction.setRedactionReason("Dossier Redaction found");
$dossierRedaction.setLegalBasis("Article 39(1)(2) of Regulation (EC) No 178/2002");
end
rule "30: Remove Dossier redactions if file is confidential"
when
FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: RedactionEntity(type == "dossier_redactions")
then
$dossierRedaction.removeFromGraph();
retract($dossierRedaction)
end
rule "101: Redact CAS Number"
when
$table: Table(hasHeader("Sample #"))
then
$table.streamTableCellsWithHeader("Sample #")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(101);
redactionEntity.setRedactionReason("Sample # found in Header");
redactionEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
insert(redactionEntity);
});
end
rule "102: Guidelines FileAttributes"
when
$section: Section(excludesTables, (containsString("DATA REQUIREMENT(S):") || containsString("TEST GUIDELINE(S):")) && (containsString("OECD") || containsString("EPA") || containsString("OPPTS")))
then
RedactionSearchUtility.findBoundariesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
end
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
rule "add NER Entities of type CBI_author"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
end
rule "combine and add NER Entities as CBI_address"
salience 999
when
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
});
end
// --------------------------------------- manual redaction rules -------------------------------------------------------------------
rule "Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$entityToBeResized: RedactionEntity(matchesAnnotationId($id))
then
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
end
rule "Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.removeFromGraph();
retract($entityToBeRemoved);
end
rule "Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: Image($id == id)
then
$entityToBeRemoved.setIgnored(true);
end
rule "Apply force redaction"
salience 128
when
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.setLegalBasis($legalBasis);
$entityToForce.setRedaction(true);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
end
rule "Apply image recategorization"
salience 128
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: Image($id == id)
then
$image.setImageType(ImageType.fromString($imageType));
end
// --------------------------------------- merging rules -------------------------------------------------------------------
rule "remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
then
$contained.removeFromGraph();
retract($contained);
end
rule "merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
then
$first.removeFromGraph();
$second.removeFromGraph();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
retract($first);
retract($second);
insert(mergedEntity);
end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then
$entity.removeFromGraph();
retract($entity)
end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$entity.addEngines($recommendation.getEngines());
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then
$recommendation.removeFromGraph();
retract($recommendation);
end
rule "remove Entity of lower rank, when equal boundaries and entityType"
salience 32
when
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
then
$lowerRank.removeFromGraph();
retract($lowerRank);
end
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
rule "remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($duplicate);
end
// --------------------------------------- local dictionary search -------------------------------------------------------------------
rule "run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> {
entity.addEngine(Engine.RULE);
insert(entity);
});
end

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -68,7 +68,7 @@ rule "Always redact CBI_author"
when
$cbiAuthor: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
then
$cbiAuthor.addMatchedRule(0);
$cbiAuthor.addMatchedRule("0");
$cbiAuthor.setRedaction(true);
$cbiAuthor.setRedactionReason("Author found");
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
@ -81,7 +81,7 @@ rule "Always redact PII"
when
$cbiAuthor: RedactionEntity(type == "PII", entityType == EntityType.ENTITY)
then
$cbiAuthor.addMatchedRule(1);
$cbiAuthor.addMatchedRule("1");
$cbiAuthor.setRedaction(true);
$cbiAuthor.setRedactionReason("PII found");
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");

48
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -78,10 +78,10 @@ rule "0: Expand CBI_author and PII entities with salutation prefix"
rule "1: Redacted because Section contains Vertebrate"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(1);
@ -92,10 +92,10 @@ rule "1: Redacted because Section contains Vertebrate"
rule "2: Not Redacted because Section contains no Vertebrate"
when
$section: Section(!hasActiveEntitiesOfType("vertebrate"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(!hasEntitiesOfType("vertebrate"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(2);
@ -105,11 +105,11 @@ rule "2: Not Redacted because Section contains no Vertebrate"
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.addMatchedRule(3);
@ -119,12 +119,12 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("no_redaction_indicator"),
hasActiveEntitiesOfType("redaction_indicator"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("no_redaction_indicator"),
hasEntitiesOfType("redaction_indicator"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.addMatchedRule(4);
@ -136,12 +136,12 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
rule "5: Do not redact Names and Addresses if published information found"
when
$section: Section(hasActiveEntitiesOfType("vertebrate"),
hasActiveEntitiesOfType("published_information"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(hasEntitiesOfType("vertebrate"),
hasEntitiesOfType("published_information"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
List<RedactionEntity> publishedInformationEntities = $section.getActiveEntitiesOfType("published_information");
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
List<RedactionEntity> publishedInformationEntities = $section.getEntitiesOfType("published_information");
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(false);
redactionEntity.setRedactionReason("Vertebrate but also Published Information found in this section");
@ -192,10 +192,10 @@ rule "7: Redact CBI_author, if its row contains a cell with header \"Vertebrate
rule "8: Redact if must_redact entity is found"
when
$section: Section(hasActiveEntitiesOfType("must_redact"),
(hasActiveEntitiesOfType("CBI_author") || hasActiveEntitiesOfType("CBI_address")))
$section: Section(hasEntitiesOfType("must_redact"),
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
then
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("must_redact entry was found.");
@ -251,7 +251,7 @@ rule "11: Redact if CTL/* or BL/* was found"
entityCreationService.byString("BL/", "must_redact", EntityType.ENTITY, $section)
.forEach(mustRedactEntity -> insert(mustRedactEntity));
$section.getActiveEntitiesOfType(List.of("CBI_author", "CBI_address"))
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
.forEach(redactionEntity -> {
redactionEntity.setRedaction(true);
redactionEntity.setRedactionReason("Laboratory for vertebrate studies found");