Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6929: fix acceptance rules/tests

Browse Source
This commit is contained in:
Kilian Schuettler 2023-07-06 17:14:29 +02:00
parent 0a47e7fae9
commit 9d7180923d
27 changed files with 850 additions and 1332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/classification/adapter/RedactionLogEntryAdapter.java
View File

@ -9,11 +9,11 @@ import java.util.Comparator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Rectangle;
@ -26,18 +26,23 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Page;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class RedactionLogEntryAdapter {
private static final double MATCH_THRESHOLD = 1;
private final EntityCreationService entityCreationService;
@Autowired
public RedactionLogEntryAdapter(EntityEnrichmentService entityEnrichmentService) {
entityCreationService = new EntityCreationService(entityEnrichmentService);
}
public Stream<RedactionEntity> toRedactionEntity(RedactionLog redactionLog, SemanticNode node) {
List<Integer> pageNumbers = redactionLog.getRedactionLogEntry().stream().flatMap(entry -> entry.getPositions().stream().map(Rectangle::getPage)).distinct().toList();
@ -72,9 +77,7 @@ public class RedactionLogEntryAdapter {
return searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
.stream()
.map(boundary -> entityCreationService.byBoundary(boundary, "temp", EntityType.ENTITY, node))
.filter(Optional::isPresent)
.map(Optional::get)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "temp", EntityType.ENTITY, node))
.collect(groupingBy(entity -> entity.getValue().toLowerCase(Locale.ROOT)));
}
@ -100,10 +103,10 @@ public class RedactionLogEntryAdapter {
private RedactionEntity createCorrectEntity(RedactionLogEntry redactionLogEntry, SemanticNode node, RedactionEntity closestEntity) {
RedactionEntity correctEntity = entityCreationService.byBoundary(closestEntity.getBoundary(),
RedactionEntity correctEntity = entityCreationService.forceByBoundary(closestEntity.getBoundary(),
redactionLogEntry.getType(),
redactionLogEntry.isRecommendation() ? EntityType.RECOMMENDATION : EntityType.ENTITY,
node).orElseThrow();
node);
String ruleIdentifier = redactionLogEntry.getType() + "." + redactionLogEntry.getMatchedRule() + ".0";
if (redactionLogEntry.isRedacted()) {
correctEntity.apply(ruleIdentifier, redactionLogEntry.getReason(), redactionLogEntry.getLegalBasis());

6
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRule.java
View File

@ -47,6 +47,12 @@ public final class MatchedRule implements Comparable<MatchedRule> {
if (Objects.equals(ruleIdentifier.type(), "MAN")) {
return -1;
}
if (Objects.equals(otherRuleIdentifier.type(), "X")) {
return 1;
}
if (Objects.equals(ruleIdentifier.type(), "X")) {
return -1;
}
}
if (!Objects.equals(otherRuleIdentifier.unit(), getRuleIdentifier().unit())) {
return otherRuleIdentifier.unit() - ruleIdentifier.unit();

48
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/MatchedRuleHolder.java
View File

@ -12,6 +12,18 @@ public interface MatchedRuleHolder {
PriorityQueue<MatchedRule> getMatchedRuleList();
boolean isIgnored();
boolean isRemoved();
void setIgnored(boolean ignored);
void setRemoved(boolean ignored);
default boolean isApplied() {
return getMatchedRule().isApplied();
@ -24,18 +36,24 @@ public interface MatchedRuleHolder {
}
default boolean isActive() {
return !(isRemoved() || isIgnored());
}
default void apply(@NonNull String ruleIdentifier, String reason, @NonNull String legalBasis) {
if (legalBasis.isBlank() || legalBasis.isEmpty()) {
throw new IllegalArgumentException("legal basis cannot be empty when redacting an entity");
}
getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).legalBasis(legalBasis).applied(true).build());
}
default void force(@NonNull String ruleIdentifier, String reason, String legalBasis) {
getMatchedRuleList().add(MatchedRule.builder()
addMatchedRule(MatchedRule.builder()
.ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier))
.reason(reason)
.legalBasis(getLegalBasisOrPreviousLegalBasisOrPlaceHolder(legalBasis))
@ -44,6 +62,26 @@ public interface MatchedRuleHolder {
}
default void skip(@NonNull String ruleIdentifier, String reason) {
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
}
default void remove(String ruleIdentifier, String reason) {
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
setRemoved(true);
}
default void ignore(String ruleIdentifier, String reason) {
addMatchedRule(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
setIgnored(true);
}
private String getLegalBasisOrPreviousLegalBasisOrPlaceHolder(String legalBasis) {
if (legalBasis == null || legalBasis.isBlank() || legalBasis.isEmpty()) {
@ -86,12 +124,6 @@ public interface MatchedRuleHolder {
}
default void skip(@NonNull String ruleIdentifier, String reason) {
getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).build());
}
default void skipWithReferences(@NonNull String ruleIdentifier, String reason, Collection<RedactionEntity> references) {
getMatchedRuleList().add(MatchedRule.builder().ruleIdentifier(RuleIdentifier.fromString(ruleIdentifier)).reason(reason).references(new HashSet<>(references)).build());

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java
View File

@ -94,12 +94,6 @@ public class RedactionEntity implements MatchedRuleHolder {
}
public boolean isActive() {
return !ignored && !removed;
}
public void addIntersectingNode(SemanticNode containingNode) {
intersectingNodes.add(containingNode);
@ -123,18 +117,6 @@ public class RedactionEntity implements MatchedRuleHolder {
}
public void remove() {
removed = true;
}
public void ignore() {
ignored = true;
}
public List<RedactionPosition> getRedactionPositionsPerPage() {
if (redactionPositionsPerPage == null || redactionPositionsPerPage.isEmpty()) {

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/Image.java
View File

@ -55,24 +55,6 @@ public class Image implements GenericSemanticNode, MatchedRuleHolder {
Set<RedactionEntity> entities = new HashSet<>();
public boolean isActive() {
return !removed && !ignored;
}
public void ignore() {
ignored = true;
}
public void remove() {
removed = true;
}
@Override
public NodeType getType() {

97
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/SemanticNode.java
View File

@ -3,6 +3,7 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.n
import static java.lang.String.format;
import java.awt.geom.Rectangle2D;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -206,6 +207,36 @@ public interface SemanticNode {
}
/**
* Checks whether this SemanticNode has any Entity of the provided types.
* Ignores Entity with ignored == true or removed == true.
*
* @param types an array of strings representing the types of entities to check for
* @return true, if this SemanticNode has at least one Entity of any of the provided types
*/
default boolean hasEntitiesOfAnyType(String... types) {
return getEntities().stream().filter(RedactionEntity::isActive).anyMatch(redactionEntity -> Arrays.stream(types).anyMatch(type -> redactionEntity.getType().equals(type)));
}
/**
* Checks whether this SemanticNode has at least one Entity of each of the provided types.
* Ignores Entity with ignored == true or removed == true.
*
* @param types an array of strings representing the types of entities to check for
* @return true, if this SemanticNode has at least one Entity of each of the provided types
*/
default boolean hasEntitiesOfAllTypes(String... types) {
return getEntities().stream()
.filter(RedactionEntity::isActive)
.map(RedactionEntity::getType)
.collect(Collectors.toUnmodifiableSet())
.containsAll(Arrays.stream(types).toList());
}
/**
* Returns a List of Entities in this SemanticNode which are of the provided type such as "CBI_author".
* Ignores Entity with ignored == true or removed == true.
@ -232,6 +263,19 @@ public interface SemanticNode {
}
/**
* Returns a List of Entities in this SemanticNode which have any of the provided types.
* Ignores Entity with the ignored flag set to true or the removed flag set to true.
*
* @param types A list of strings representing the types of entities to return
* @return List of RedactionEntities that match any of the provided types
*/
default List<RedactionEntity> getEntitiesOfType(String... types) {
return getEntities().stream().filter(RedactionEntity::isActive).filter(redactionEntity -> redactionEntity.isAnyType(Arrays.stream(types).toList())).toList();
}
/**
* Each AtomicTextBlock has an index on its page, this returns the number of the first AtomicTextBlock underneath this node.
* If this node does not have any AtomicTexBlocks underneath it, e.g. an empty TableCell. It returns -1.
@ -278,21 +322,21 @@ public interface SemanticNode {
* @param strings A List of Strings which the TextBlock might contain
* @return true, if this node's TextBlock contains all strings
*/
default boolean containsStrings(List<String> strings) {
default boolean containsAllStrings(String... strings) {
return strings.stream().allMatch(this::containsString);
return Arrays.stream(strings).allMatch(this::containsString);
}
/**
* Checks whether this SemanticNode contains all the provided Strings ignoring case.
* Checks whether this SemanticNode contains any of the provided Strings.
*
* @param string A String which the TextBlock might contain
* @return true, if this node's TextBlock contains the string ignoring case
* @param strings A List of Strings to check if they are contained in the TextBlock
* @return true, if this node's TextBlock contains any of the provided strings
*/
default boolean containsStringIgnoreCase(String string) {
default boolean containsAnyString(String... strings) {
return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
return Arrays.stream(strings).anyMatch(this::containsString);
}
@ -308,15 +352,39 @@ public interface SemanticNode {
}
/**
* Checks whether this SemanticNode contains all the provided Strings ignoring case.
*
* @param string A String which the TextBlock might contain
* @return true, if this node's TextBlock contains the string ignoring case
*/
default boolean containsStringIgnoreCase(String string) {
return getTextBlock().getSearchText().toLowerCase().contains(string.toLowerCase());
}
/**
* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
*
* @param strings A List of Strings which the TextBlock might contain
* @return true, if this node's TextBlock contains any of the strings
*/
default boolean containsAnyStringIgnoreCase(List<String> strings) {
default boolean containsAnyStringIgnoreCase(String... strings) {
return strings.stream().anyMatch(this::containsStringIgnoreCase);
return Arrays.stream(strings).anyMatch(this::containsStringIgnoreCase);
}
/**
* Checks whether this SemanticNode contains any of the provided Strings ignoring case.
*
* @param strings A List of Strings which the TextBlock might contain
* @return true, if this node's TextBlock contains any of the strings
*/
default boolean containsAllStringsIgnoreCase(String... strings) {
return Arrays.stream(strings).allMatch(this::containsStringIgnoreCase);
}
@ -420,6 +488,17 @@ public interface SemanticNode {
}
/**
* Returns the length of the text content in this Node's TextBlock.
*
* @return The length of the text content
*/
default int length() {
return getBoundary().length();
}
/**
* If this Node is a Leaf it will calculate the boundingBox of its LeafTextBlock, otherwise it will calculate the Union of the BoundingBoxes of all its Children.
* If called on the Document, it will return the cropbox of each page

55
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java
View File

@ -15,7 +15,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.springframework.stereotype.Service;
import org.kie.api.runtime.KieSession;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
@ -37,11 +37,18 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Service
@RequiredArgsConstructor
public class EntityCreationService {
private final EntityEnrichmentService entityEnrichmentService;
private final KieSession kieSession;
public EntityCreationService(EntityEnrichmentService entityEnrichmentService) {
this.entityEnrichmentService = entityEnrichmentService;
this.kieSession = null;
}
public Stream<RedactionEntity> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
@ -138,22 +145,6 @@ public class EntityCreationService {
}
public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation,
String type,
EntityType entityType,
SemanticNode node,
boolean isDossierDictionaryEntry) {
searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> isValidEntityBoundary(node.getTextBlock(), boundary))
.map(bounds -> forceByBoundary(bounds, type, entityType, node))
.peek(entity -> entity.setDictionaryEntry(true))
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
.forEach(entity -> entity.addEngine(Engine.DICTIONARY));
}
public Stream<RedactionEntity> lineAfterStrings(List<String> strings, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.getTextBlock();
@ -181,6 +172,21 @@ public class EntityCreationService {
}
public Optional<RedactionEntity> semanticNodeAfterString(SemanticNode semanticNode, String string, String type, EntityType entityType) {
var textBlock = semanticNode.getTextBlock();
int startIndex = Math.min(textBlock.indexOf(string), 0);
var boundary = new Boundary(startIndex, semanticNode.getBoundary().end());
if (boundary.length() > 0) {
boundary = new Boundary(boundary.start(), boundary.end() - 1);
}
if (!isValidEntityBoundary(textBlock, boundary)) {
return Optional.empty();
}
return byBoundary(boundary, type, entityType, semanticNode);
}
public Stream<RedactionEntity> byRegexWithLineBreaks(String regexPattern, String type, EntityType entityType, SemanticNode node) {
return byRegexWithLineBreaks(regexPattern, type, entityType, 0, node);
@ -301,13 +307,14 @@ public class EntityCreationService {
/**
* Creates a redaction entity based on the given boundary, type, entity type, and semantic node.
* If the document already contains an equal redaction entity, then en empty Optional is returned.
* If the document already contains an equal redaction entity, then the original Entity is returned.
* Also inserts the Entity into the kieSession.
*
* @param boundary The boundary of the redaction entity.
* @param type The type of the redaction entity.
* @param entityType The entity type of the redaction entity.
* @param node The semantic node to associate with the redaction entity.
* @return An Optional containing the redaction entity, or an empty Optional if the entity already exists.
* @return An Optional containing the redaction entity, or the previous entity if the entity already exists.
*/
public Optional<RedactionEntity> byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
@ -317,10 +324,13 @@ public class EntityCreationService {
Boundary trimmedBoundary = boundary.trim(node.getTextBlock());
RedactionEntity entity = RedactionEntity.initialEntityNode(trimmedBoundary, type, entityType);
if (node.getEntities().contains(entity)) {
return Optional.empty();
return node.getEntities().stream().filter(entity::equals).peek(e -> e.addEngine(Engine.RULE)).findAny();
}
addEntityToGraph(entity, node);
entity.addEngine(Engine.RULE);
if (kieSession != null) {
kieSession.insert(entity);
}
return Optional.of(entity);
}
@ -356,6 +366,7 @@ public class EntityCreationService {
mergedEntity.setRemoved(entitiesToMerge.stream().allMatch(RedactionEntity::isRemoved));
addEntityToGraph(mergedEntity, node);
kieSession.insert(mergedEntity);
return mergedEntity;
}
@ -391,7 +402,7 @@ public class EntityCreationService {
}
private boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
public boolean isValidEntityBoundary(TextBlock textBlock, Boundary boundary) {
return boundary.length() > 0 && boundaryIsSurroundedBySeparators(textBlock, boundary);
}

15
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/ManualRedactionApplicationService.java
View File

@ -3,16 +3,15 @@ package com.iqser.red.service.redaction.v1.server.layoutparsing.document.service
import java.awt.geom.Rectangle2D;
import java.util.NoSuchElementException;
import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Image;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RectangleTransformations;
import lombok.RequiredArgsConstructor;
@Service
@RequiredArgsConstructor
public class ManualRedactionApplicationService {
@ -47,6 +46,16 @@ public class ManualRedactionApplicationService {
}
public void resizeImage(Image image, ManualResizeRedaction manualResizeRedaction) {
if (manualResizeRedaction.getPositions().isEmpty() || manualResizeRedaction.getPositions() == null) {
return;
}
var bBox = RectangleTransformations.rectangle2DBBox(manualResizeRedaction.getPositions().stream().map(ManualRedactionApplicationService::toRectangle2D).toList());
image.setPosition(bBox);
}
private static Rectangle2D toRectangle2D(com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.Rectangle rect) {
return new Rectangle2D.Double(rect.getTopLeftX() - rect.getWidth(), rect.getTopLeftY() - rect.getHeight(), rect.getWidth(), rect.getHeight());

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/AnalyzeService.java
View File

@ -138,7 +138,7 @@ public class AnalyzeService {
dictionaryService.updateDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
log.info("Updated Dictionary for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
log.info("Updated Dictionaries for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
long rulesVersion = droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId());
@ -180,8 +180,10 @@ public class AnalyzeService {
long startTime = System.currentTimeMillis();
RedactionLog previousRedactionLog = redactionStorageService.getRedactionLog(analyzeRequest.getDossierId(), analyzeRequest.getFileId());
log.info("Loaded previous redaction log for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(analyzeRequest.getDossierId(), analyzeRequest.getFileId()));
log.info("Loaded Document Graph for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
// not yet ready for reanalysis
if (previousRedactionLog == null || document == null || document.getNumberOfPages() == 0) {
return analyze(analyzeRequest);
@ -207,10 +209,14 @@ public class AnalyzeService {
NerEntities nerEntities = getEntityRecognitionEntitiesFilteredBySectionIds(analyzeRequest, document, sectionsToReanalyseIds);
log.info("Loaded Ner Entities for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
KieContainer kieContainer = droolsExecutionService.updateRules(analyzeRequest.getDossierTemplateId());
log.info("Updated Rules to version {} for file {} in dossier {}",
droolsExecutionService.getRulesVersion(analyzeRequest.getDossierTemplateId()),
analyzeRequest.getFileId(),
analyzeRequest.getDossierId());
Dictionary dictionary = dictionaryService.getDeepCopyDictionary(analyzeRequest.getDossierTemplateId(), analyzeRequest.getDossierId());
sectionsToReAnalyse.forEach(node -> entityRedactionService.addDictionaryEntities(dictionary, node));
log.info("Finished Dictionary Search for file {} in dossier {}", analyzeRequest.getFileId(), analyzeRequest.getDossierId());

7
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DroolsExecutionService.java
View File

@ -26,6 +26,7 @@ import com.iqser.red.service.redaction.v1.server.exception.RulesValidationExcept
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
@ -46,8 +47,7 @@ public class DroolsExecutionService {
RulesClient rulesClient;
Map<String, KieContainer> kieContainers = new HashMap<>();
Map<String, Long> rulesVersionPerDossierTemplateId = new HashMap<>();
EntityCreationService entityCreationService;
ManualRedactionApplicationService manualRedactionApplicationService;
EntityEnrichmentService entityEnrichmentService;
NerEntitiesAdapter nerEntitiesAdapter;
@ -84,6 +84,9 @@ public class DroolsExecutionService {
NerEntities nerEntities) {
KieSession kieSession = kieContainer.newKieSession();
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
ManualRedactionApplicationService manualRedactionApplicationService = new ManualRedactionApplicationService(entityCreationService);
kieSession.setGlobal("document", document);
kieSession.setGlobal("entityCreationService", entityCreationService);
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);

32
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java
View File

@ -9,12 +9,15 @@ import org.springframework.stereotype.Service;
import com.iqser.red.service.persistence.service.v1.api.shared.model.AnalyzeRequest;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.SearchImplementation;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
@ -28,7 +31,7 @@ import lombok.extern.slf4j.Slf4j;
public class EntityRedactionService {
DroolsExecutionService droolsExecutionService;
EntityCreationService entityCreationService;
EntityEnrichmentService entityEnrichmentService;
public Set<FileAttribute> addRuleEntities(Dictionary dictionary, Document document, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
@ -71,14 +74,27 @@ public class EntityRedactionService {
public void addDictionaryEntities(Dictionary dictionary, SemanticNode node) {
for (var model : dictionary.getDictionaryModels()) {
entityCreationService.bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
entityCreationService.bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
entityCreationService.bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(),
model.getType(),
EntityType.FALSE_RECOMMENDATION,
node,
model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getEntriesSearch(), model.getType(), EntityType.ENTITY, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalsePositiveSearch(), model.getType(), EntityType.FALSE_POSITIVE, node, model.isDossierDictionary());
bySearchImplementationAsDictionary(model.getFalseRecommendationsSearch(), model.getType(), EntityType.FALSE_RECOMMENDATION, node, model.isDossierDictionary());
}
}
public void bySearchImplementationAsDictionary(SearchImplementation searchImplementation,
String type,
EntityType entityType,
SemanticNode node,
boolean isDossierDictionaryEntry) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
searchImplementation.getBoundaries(node.getTextBlock(), node.getBoundary())
.stream()
.filter(boundary -> entityCreationService.isValidEntityBoundary(node.getTextBlock(), boundary))
.map(bounds -> entityCreationService.forceByBoundary(bounds, type, entityType, node))
.peek(entity -> entity.setDictionaryEntry(true))
.peek(entity -> entity.setDossierDictionaryEntry(isDossierDictionaryEntry))
.forEach(entity -> entity.addEngine(Engine.DICTIONARY));
}
}

4
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/ManualRedactionSurroundingTextService.java
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
@ -35,7 +36,7 @@ import lombok.extern.slf4j.Slf4j;
public class ManualRedactionSurroundingTextService {
private final RedactionStorageService redactionStorageService;
private final EntityCreationService entityCreationService;
private final EntityEnrichmentService entityEnrichmentService;
@Timed("redactmanager_surroundingTextAnalysis")
@ -73,6 +74,7 @@ public class ManualRedactionSurroundingTextService {
private Pair<String, String> findSurroundingText(SemanticNode node, String value, List<Rectangle> toFindPositions) {
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
Set<RedactionEntity> entities = RedactionSearchUtility.findBoundariesByString(value, node.getTextBlock())
.stream()
.map(boundary -> entityCreationService.forceByBoundary(boundary, "searchHelper", EntityType.RECOMMENDATION, node))

2
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/RedactionChangeLogService.java
View File

@ -83,7 +83,7 @@ public class RedactionChangeLogService {
currentRedactionLog.setRedactionLogEntry(newRedactionLogEntries);
log.info("Change computation took: {}", System.currentTimeMillis() - start);
log.debug("Change computation took: {}", System.currentTimeMillis() - start);
return new RedactionLogChanges(currentRedactionLog, !addedEntryIds.isEmpty() || !removedIds.isEmpty());
}

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumineFloraTest.java
View File

@ -42,6 +42,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
@Test
@Disabled
public void titleExtraction() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/Documine/Flora/A13617AV/425_F.1.1.1 - A13617AV - Acute Oral Toxicity Study.pdf");
@ -65,6 +66,7 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
@Test
@Disabled
public void tableWithEmptyCols() throws IOException {
// FIXME TableNodeFactory: 36, why has table no rows/cols here.
@ -89,8 +91,8 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
}
@Test
@Disabled
public void testTopOfPage13InNotHeader() throws IOException {
// Fix In BodyTextFrameService destroys header detection in files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf
@ -116,7 +118,6 @@ public class DocumineFloraTest extends AbstractRedactionIntegrationTest {
}
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ComponentScan(excludeFilters = {@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = StorageAutoConfiguration.class)})

88
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -8,6 +8,7 @@ import java.io.IOException;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@ -30,6 +31,8 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLog;
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.RedactionLogEntry;
import com.iqser.red.service.redaction.v1.model.StructureAnalyzeRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateRequest;
import com.iqser.red.service.redaction.v1.server.annotate.AnnotateResponse;
@ -103,29 +106,12 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
System.out.println("Finished analysis");
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry1 = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals("published_information"))
.filter(entry -> entry.getValue().equals("Oxford University Press"))
.findFirst()
.orElseThrow();
var asyaLyon1 = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals("CBI_author"))
.filter(entry -> entry.getValue().equals("Asya Lyon"))
.filter(entry -> entry.getSectionNumber() == publishedInformationEntry1.getSectionNumber())
.findFirst()
.orElseThrow();
var publishedInformationEntry1 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
var asyaLyon1 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry1.getSectionNumber()).findFirst().orElseThrow();
// assertFalse(asyaLyon1.isRedacted());
var idRemoval = IdRemoval.builder()
.requestDate(OffsetDateTime.now())
.annotationId(publishedInformationEntry1.getId())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.build();
var idRemoval = buildIdRemoval(publishedInformationEntry1.getId());
var manualRedactions = ManualRedactions.builder().idsToRemove(Set.of(idRemoval)).build();
request.setManualRedactions(manualRedactions);
@ -133,20 +119,8 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var publishedInformationEntry2 = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals("published_information"))
.filter(entry -> entry.getValue().equals("Oxford University Press"))
.findFirst()
.orElseThrow();
var asyaLyon2 = redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals("CBI_author"))
.filter(entry -> entry.getValue().equals("Asya Lyon"))
.filter(entry -> entry.getSectionNumber() == publishedInformationEntry2.getSectionNumber())
.findFirst()
.orElseThrow();
var publishedInformationEntry2 = findEntityByTypeAndValue(redactionLog, "published_information", "Oxford University Press").findFirst().orElseThrow();
var asyaLyon2 = findEntityByTypeAndValueAndSectionNumber(redactionLog, "CBI_author", "Asya Lyon", publishedInformationEntry2.getSectionNumber()).findFirst().orElseThrow();
assertTrue(asyaLyon2.isRedacted());
@ -160,4 +134,50 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
}
private Stream<RedactionLogEntry> findEntityByTypeAndValueAndSectionNumber(RedactionLog redactionLog, String type, String value, int sectionNumber) {
return redactionLog.getRedactionLogEntry()
.stream()
.filter(entry -> entry.getType().equals(type))
.filter(entry -> entry.getValue().equals(value))
.filter(entry -> entry.getSectionNumber() == sectionNumber);
}
private static Stream<RedactionLogEntry> findEntityByTypeAndValue(RedactionLog redactionLog, String type, String value) {
return redactionLog.getRedactionLogEntry().stream().filter(entry -> entry.getType().equals(type)).filter(entry -> entry.getValue().equals(value));
}
@Test
public void noEndlessLoopsTest() {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1_moreSections.pdf");
System.out.println("Start Full integration test");
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
System.out.println("Finished structure analysis");
AnalyzeResult result = analyzeService.analyze(request);
var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
RedactionLogEntry desireeEtAl = findEntityByTypeAndValue(redactionLog, "CBI_author", "Desiree").filter(e -> e.getMatchedRule().startsWith("CBI.16"))
.findAny()
.orElseThrow();
IdRemoval removal = buildIdRemoval(desireeEtAl.getId());
request.setManualRedactions(ManualRedactions.builder().idsToRemove(Set.of(removal)).build());
analyzeService.reanalyze(request);
var redactionLog2 = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
assertTrue(findEntityByTypeAndValue(redactionLog2, "CBI_author", "Desiree").noneMatch(e -> e.getMatchedRule().startsWith("CBI.16")));
}
private static IdRemoval buildIdRemoval(String id) {
return IdRemoval.builder().annotationId(id).requestDate(OffsetDateTime.now()).status(AnnotationStatus.APPROVED).fileId(TEST_FILE_ID).build();
}
}

5
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -75,6 +75,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Section;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.redaction.utils.OsUtils;
import com.iqser.red.service.redaction.v1.server.storage.RedactionStorageService;
import com.iqser.red.storage.commons.StorageAutoConfiguration;
@ -89,9 +90,8 @@ import lombok.SneakyThrows;
public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
@Autowired
private EntityCreationService entityCreationService;
private EntityEnrichmentService entityEnrichmentService;
@Configuration
@EnableAutoConfiguration(exclude = {RabbitAutoConfiguration.class})
@ -702,6 +702,7 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
Document document = DocumentGraphMapper.toDocumentGraph(redactionStorageService.getDocumentData(TEST_DOSSIER_ID, TEST_FILE_ID));
String expandedEntityKeyword = "Lorem ipsum dolor sit amet, consectetur adipiscing elit Desiree et al sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Melanie et al. Reference No 12345 Lorem ipsum.";
EntityCreationService entityCreationService = new EntityCreationService(entityEnrichmentService);
RedactionEntity expandedEntity = entityCreationService.byString(expandedEntityKeyword, "PII", EntityType.ENTITY, document).findFirst().get();
String idToResize = redactionLog.getRedactionLogEntry()

23
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentEntityInsertionIntegrationTest.java
View File

@ -2,11 +2,18 @@ package com.iqser.red.service.redaction.v1.server.document.graph;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.wildfly.common.Assert.assertTrue;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.kie.api.runtime.KieSession;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.beans.factory.annotation.Autowired;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
@ -23,12 +30,25 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegrationTest {
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
@Mock
private KieSession kieSession;
@BeforeEach
public void createEntityCreationService() {
MockitoAnnotations.initMocks(this);
entityCreationService = new EntityCreationService(entityEnrichmentService, kieSession);
}
@Test
public void assertCollectAllEntitiesWorks() {
@ -50,8 +70,9 @@ public class DocumentEntityInsertionIntegrationTest extends BuildDocumentIntegra
Document document = buildGraph("files/new/crafted document.pdf");
String type = "CBI_author";
assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent());
assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isEmpty());
assertTrue(entityCreationService.byBoundary(new Boundary(0, 10), type, EntityType.ENTITY, document).isPresent());
assertEquals(1, document.getEntities().size());
verify(kieSession, times(1)).insert(any(RedactionEntity.class));
}

6
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/services/DocumentPerformanceIntegrationTest.java → redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -1,4 +1,4 @@
package com.iqser.red.service.redaction.v1.server.document.services;
package com.iqser.red.service.redaction.v1.server.document.graph;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.boundaryIsSurroundedBySeparators;
import static org.mockito.Mockito.when;
@ -31,7 +31,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.ManualRedactions;
import com.iqser.red.service.persistence.service.v1.api.shared.model.common.JSONPrimitive;
import com.iqser.red.service.persistence.service.v1.api.shared.model.dossiertemplate.type.Type;
import com.iqser.red.service.redaction.v1.server.document.graph.BuildDocumentIntegrationTest;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
@ -40,6 +39,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.no
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.PdfVisualisationUtility;
import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
@ -60,6 +60,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
private DictionaryService dictionaryService;
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
@Autowired
@ -93,6 +94,7 @@ public class DocumentPerformanceIntegrationTest extends BuildDocumentIntegration
@BeforeEach
public void stubClients() {
entityCreationService = new EntityCreationService(entityEnrichmentService);
TenantContext.setTenantId("redaction");
when(rulesClient.getVersion(TEST_DOSSIER_TEMPLATE_ID)).thenReturn(0L);

23
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/ManualResizeRedactionIntegrationTest.java
View File

@ -11,6 +11,7 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.kie.api.KieServices;
import org.kie.api.builder.KieBuilder;
@ -36,6 +37,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
@Import(ManualResizeRedactionIntegrationTest.TestConfiguration.class)
@ -44,9 +46,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
private static final String RULES = "drools/manual_redaction_rules.drl";
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
@Autowired
private ManualRedactionApplicationService manualRedactionApplicationService;
@Qualifier("kieContainer")
@ -74,6 +75,14 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
}
@BeforeEach
public void createServices() {
entityCreationService = new EntityCreationService(entityEnrichmentService);
manualRedactionApplicationService = new ManualRedactionApplicationService(entityCreationService);
}
@Test
public void manualResizeRedactionTest() {
@ -95,6 +104,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(manualResizeRedaction);
kieSession.fireAllRules();
@ -131,6 +142,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.insert(entity);
kieSession.insert(manualForceRedaction);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.fireAllRules();
kieSession.dispose();
@ -159,6 +172,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.fireAllRules();
@ -189,6 +204,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
KieSession kieSession = kieContainer.newKieSession();
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.insert(manualForceRedaction);
@ -220,6 +237,8 @@ public class ManualResizeRedactionIntegrationTest extends BuildDocumentIntegrati
kieSession.setGlobal("manualRedactionApplicationService", manualRedactionApplicationService);
kieSession.insert(entity);
kieSession.insert(idRemoval);
kieSession.insert(document);
document.streamAllSubNodes().forEach(kieSession::insert);
kieSession.fireAllRules();
kieSession.dispose();

10
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/adapter/NerEntitiesAdapterTest.java
View File

@ -13,6 +13,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.ClassPathResource;
@ -26,6 +27,7 @@ import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.en
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionPosition;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.Document;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityEnrichmentService;
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.PdfVisualisationUtility;
import lombok.SneakyThrows;
@ -37,6 +39,7 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
@Autowired
private NerEntitiesAdapter nerEntitiesAdapter;
@Autowired
private EntityEnrichmentService entityEnrichmentService;
private EntityCreationService entityCreationService;
@ -48,6 +51,13 @@ class NerEntitiesAdapterTest extends BuildDocumentIntegrationTest {
}
@BeforeEach
public void createEntityCreationService() {
entityCreationService = new EntityCreationService(entityEnrichmentService);
}
@Test
@SneakyThrows
public void testGetNerEntities() {

158
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -64,10 +64,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
then
/* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("SYN.1.0", "");
insert(entity);
});
.forEach(entity -> entity.skip("SYN.1.0", ""));
end
@ -115,10 +112,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
$entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
then
entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> {
falsePositive.skip("CBI.2.0", "Genitive Author found");
insert(falsePositive);
});
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
end
@ -166,10 +160,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
@ -182,10 +173,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -200,10 +188,7 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
@ -216,10 +201,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -245,7 +227,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
end
@ -258,7 +239,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
@ -270,10 +250,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
end
rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
@ -281,10 +258,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
end
@ -299,7 +273,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -313,7 +286,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -345,10 +317,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@ -357,10 +326,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -380,10 +346,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
@ -401,10 +364,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -415,10 +375,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
@ -427,10 +384,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
@ -439,10 +393,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
@ -451,10 +402,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -466,7 +414,7 @@ rule "ETC.0.0: Purity Hint"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", ""));
.forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end
@ -479,14 +427,6 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.2.0: Redact signatures (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
then
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: ETC.3
rule "ETC.3.0: Redact logos (vertebrate study)"
@ -512,7 +452,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: RedactionEntity(type == "dossier_redaction")
then
$dossierRedaction.setIgnored(true);
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -527,8 +467,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -540,10 +479,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
});
.forEach(entity -> entity.addEngine(Engine.NER));
end
@ -571,7 +507,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.setIgnored(true);
$entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction");
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
@ -584,7 +520,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.setIgnored(true);
$imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction");
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
@ -593,6 +529,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
@ -604,7 +541,6 @@ rule "MAN.2.0: Apply force redaction"
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
@ -628,10 +564,10 @@ rule "MAN.3.0: Apply image recategorization"
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
$larger: RedactionEntity($type: type, $entityType: entityType, isActive())
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$contained.remove();
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
@ -640,15 +576,14 @@ rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$first.remove();
$second.remove();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
$second.remove("X.1.0", "merge intersecting Entities of same type");
retract($first);
retract($second);
insert(mergedEntity);
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
end
@ -657,11 +592,11 @@ rule "X.1.0: merge intersecting Entities of same type"
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive())
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove();
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -670,10 +605,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive())
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
@ -682,11 +617,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove();
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
@ -695,10 +630,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity(entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
@ -707,11 +642,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
salience 32
when
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove();
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
retract($lowerRank);
end
@ -738,6 +673,5 @@ rule "LDS.0.0: run local dictionary search"
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> insert(entity));
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList();
end

296
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_rules.drl
View File

@ -58,7 +58,7 @@ query "getFileAttributes"
//------------------------------------ Syngenta specific rules ------------------------------------
// Rule unit: SYN.0
rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("CTL/") || containsString("BL/"))
@ -66,10 +66,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
).forEach(entity -> {
entity.skip("SYN.0.0", "hint_only");
insert(entity);
});
).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end
@ -80,10 +77,7 @@ rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
then
/* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("SYN.1.0", "");
insert(entity);
});
.forEach(entity -> entity.skip("SYN.1.0", ""));
end
@ -131,10 +125,7 @@ rule "CBI.2.0: Don't redact genitive CBI_author"
$entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), isApplied())
then
entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document)
.ifPresent(falsePositive -> {
falsePositive.skip("CBI.2.0", "Genitive Author found");
insert(falsePositive);
});
.ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
end
@ -385,10 +376,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
@ -401,10 +389,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -419,10 +404,7 @@ rule "CBI.10.0: Redact all Cell's with Header Author(s) as CBI_author (vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.0", "Author(s) found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate study)"
@ -435,10 +417,7 @@ rule "CBI.10.1: Redact all Cell's with Header Author as CBI_author (vertebrate s
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.10.1", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -466,10 +445,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.skip("CBI.12.0", "Author(s) header found");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
end
rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
@ -478,9 +454,7 @@ rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study");
});
.forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"));
end
rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
@ -489,9 +463,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -501,7 +473,7 @@ rule "CBI.13.0: Ignore CBI Address Recommendations"
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$entity: RedactionEntity(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
then
$entity.remove();
$entity.ignore("CBI.13.0", "Ignore CBI Address Recommendations");
retract($entity)
end
@ -533,12 +505,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
.forEach(keywordEntity -> insert(keywordEntity));
.toList();
$section.getEntitiesOfType(List.of($keyword, $residueKeyword))
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
@ -556,13 +526,11 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
$table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
.forEach(keywordEntity -> insert(keywordEntity));
.toList();
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -577,7 +545,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
end
@ -590,7 +557,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
@ -602,10 +568,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
end
rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
@ -613,10 +576,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
end
@ -632,11 +592,10 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
then
entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
$entityToExpand.remove();
expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList());
$entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials");
retract($entityToExpand);
insert(expandedEntity);
});
});
end
@ -647,11 +606,10 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
$entityToExpand.remove();
expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList());
$entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix");
retract($entityToExpand);
insert(expandedEntity);
});
});
end
@ -666,7 +624,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -680,7 +637,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -712,10 +668,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@ -724,10 +677,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -747,10 +697,7 @@ rule "PII.2.0: Redact Phone and Fax by RegEx (non vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.2.0", "Found by Phone and Fax Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
@ -768,10 +715,7 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
containsString("Fer"))
then
entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.2.1", "Found by Phone and Fax Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -782,10 +726,7 @@ rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
@ -794,10 +735,7 @@ rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -827,10 +765,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)"
@ -858,10 +793,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -876,10 +808,7 @@ rule "PII.5.0: Redact line after contact information keywords reduced (non verte
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
@ -892,10 +821,7 @@ rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrat
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -909,10 +835,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.1: redact line between contact keywords"
@ -924,10 +847,7 @@ rule "PII.6.1: redact line between contact keywords"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -947,10 +867,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
@ -968,10 +885,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -991,10 +905,7 @@ rule "PII.8.0: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(entity);
});
.forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.8.1: Redact contact information if producer is found"
@ -1012,10 +923,7 @@ rule "PII.8.1: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -1026,10 +934,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
@ -1038,10 +943,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
@ -1050,10 +952,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
@ -1062,10 +961,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -1075,10 +971,7 @@ rule "PII.10.0: Redact study director abbreviation"
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -1088,10 +981,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -1101,10 +991,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
$entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
insert(expandedEntity);
});
.ifPresent(expandedEntity -> expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()));
end
@ -1116,7 +1003,7 @@ rule "ETC.0.0: Purity Hint"
$section: Section(containsStringIgnoreCase("purity"))
then
entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
.forEach(hint -> hint.skip("ETC.0.0", ""));
.forEach(hint -> hint.skip("ETC.0.0", "hint only"));
end
@ -1139,7 +1026,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.2.0: Redact signatures (vertebrate study)"
rule "ETC.2.1: Redact signatures (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
@ -1181,7 +1068,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: RedactionEntity(type == "dossier_redaction")
then
$dossierRedaction.setIgnored(true);
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -1196,10 +1083,7 @@ rule "ETC.6.0: Redact CAS Number"
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -1242,8 +1126,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -1255,10 +1138,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
});
.forEach(entity -> entity.addEngine(Engine.NER));
end
@ -1270,9 +1150,7 @@ rule "AI.2.0: add all NER Entities of any type except CBI_author"
then
nerEntities.getNerEntityList().stream()
.filter(nerEntity -> !nerEntity.type().equals("CBI_author"))
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, nerEntity.type().toLowerCase(), EntityType.RECOMMENDATION, document));
end
@ -1291,6 +1169,18 @@ rule "MAN.0.0: Apply manual resize redaction"
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$imageToBeResized: Image(id == $id)
then
manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
@ -1300,7 +1190,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.setIgnored(true);
$entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction");
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
@ -1313,7 +1203,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.setIgnored(true);
$imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction");
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
@ -1322,6 +1212,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
@ -1333,7 +1224,6 @@ rule "MAN.2.0: Apply force redaction"
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
@ -1370,10 +1260,10 @@ rule "MAN.3.0: Apply image recategorization"
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
$larger: RedactionEntity($type: type, $entityType: entityType, isActive())
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$contained.remove();
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
@ -1382,15 +1272,14 @@ rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$first.remove();
$second.remove();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
$second.remove("X.1.0", "merge intersecting Entities of same type");
retract($first);
retract($second);
insert(mergedEntity);
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
end
@ -1399,11 +1288,11 @@ rule "X.1.0: merge intersecting Entities of same type"
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive())
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove();
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1412,10 +1301,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive())
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
@ -1424,11 +1313,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove();
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
@ -1437,10 +1326,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity(entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
@ -1449,11 +1338,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
salience 32
when
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove();
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
retract($lowerRank);
end
@ -1480,6 +1369,5 @@ rule "LDS.0.0: run local dictionary search"
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> insert(entity));
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList();
end

58
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -58,6 +58,19 @@ rule "MAN.0.0: Apply manual resize redaction"
manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$imageToBeResized: Image(id == $id)
then
manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
@ -65,44 +78,71 @@ rule "MAN.0.0: Apply manual resize redaction"
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
$idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.setIgnored(true);
$entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction");
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
$idRemoval: IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.setIgnored(true);
$imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction");
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
end
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$entityToForce: RedactionEntity(matchesAnnotationId($id))
then
$entityToForce.apply("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$entityToForce.setRemoved(false);
$entityToForce.setIgnored(false);
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$imageToForce: Image(id == $id)
then
$imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$imageToForce.setRemoved(false);
$imageToForce.setIgnored(false);
update($imageToForce);
update($imageToForce.getParent());
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply image recategorization"
salience 128
when
ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$image: Image($id == id)
$recategorization: ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
$imageToBeRecategorized: Image($id == id)
then
$image.setImageType(ImageType.fromString($imageType));
$imageToBeRecategorized.setImageType(ImageType.fromString($imageType));
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
//------------------------------------ Local dictionary search rules ------------------------------------

247
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -58,7 +58,7 @@ query "getFileAttributes"
//------------------------------------ Syngenta specific rules ------------------------------------
// Rule unit: SYN.0
rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
when
not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
$section: Section(containsString("CTL/") || containsString("BL/"))
@ -66,10 +66,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
Stream.concat(
entityCreationService.byString("CTL", "must_redact", EntityType.ENTITY, $section),
entityCreationService.byString("BL", "must_redact", EntityType.ENTITY, $section)
).forEach(entity -> {
entity.skip("SYN.0.0", "hint_only");
insert(entity);
});
).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
end
@ -249,10 +246,7 @@ rule "CBI.9.0: Redact all Cell's with Header Author(s) as CBI_author (non verteb
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrate study)"
@ -265,10 +259,7 @@ rule "CBI.9.1: Redact all Cell's with Header Author as CBI_author (non vertebrat
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -296,10 +287,7 @@ rule "CBI.12.0: Add all Cell's with Header Author(s) as CBI_author"
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.skip("CBI.12.0", "Author(s) header found");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.skip("CBI.12.0", "Author(s) header found"));
end
rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value No"
@ -308,9 +296,7 @@ rule "CBI.12.1: Dont redact CBI_author, if its row contains a cell with header \
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("N", "No"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study");
});
.forEach(authorEntity -> authorEntity.skip("CBI.12.1", "Not redacted because it's row does not belong to a vertebrate study"));
end
rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vertebrate study Y/N\" and value Yes"
@ -319,9 +305,7 @@ rule "CBI.12.2: Redact CBI_author, if its row contains a cell with header \"Vert
then
$table.streamEntitiesWhereRowHasHeaderAndAnyValue("Vertebrate study Y/N", List.of("Y", "Yes"))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(authorEntity -> {
authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(authorEntity -> authorEntity.apply("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -352,12 +336,10 @@ rule "CBI.15.0: Redact row if row contains \"determination of residues\" and liv
containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $section)
.forEach(keywordEntity -> insert(keywordEntity));
.toList();
$section.getEntitiesOfType(List.of($keyword, $residueKeyword))
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
@ -375,13 +357,11 @@ rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determinatio
$table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword))
then
entityCreationService.byString($keyword, "must_redact", EntityType.ENTITY, $table)
.forEach(keywordEntity -> insert(keywordEntity));
.toList();
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
.forEach(redactionEntity -> {
redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
});
.forEach(redactionEntity -> redactionEntity.apply("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -396,7 +376,6 @@ rule "CBI.16.0: Add CBI_author with \"et al.\" Regex (non vertebrate study)"
.forEach(entity -> {
entity.apply("CBI.16.0", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
insert(entity);
});
end
@ -409,7 +388,6 @@ rule "CBI.16.1: Add CBI_author with \"et al.\" Regex (vertebrate study)"
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
.forEach(entity -> {
entity.apply("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
});
end
@ -421,10 +399,7 @@ rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
end
rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
@ -432,10 +407,7 @@ rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with
$section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
then
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
.forEach(entity -> {
entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section");
insert(entity);
});
.forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
end
@ -451,11 +423,10 @@ rule "CBI.18.0: Expand CBI_author entities with firstname initials"
then
entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
$entityToExpand.remove();
expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList());
$entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials");
retract($entityToExpand);
insert(expandedEntity);
});
});
end
@ -466,11 +437,10 @@ rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
$entityToExpand.remove();
expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList());
$entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix");
retract($entityToExpand);
insert(expandedEntity);
});
});
end
@ -485,7 +455,6 @@ rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.skip("CBI.20.0", "PERFORMING LABORATORY was found for non vertebrate study");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -499,7 +468,6 @@ rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
.forEach(laboratoryEntity -> {
laboratoryEntity.apply("CBI.20.1", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
dictionary.addLocalDictionaryEntry(laboratoryEntity);
insert(laboratoryEntity);
});
end
@ -531,10 +499,7 @@ rule "PII.1.0: Redact Emails by RegEx (Non vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.0", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
@ -543,10 +508,7 @@ rule "PII.1.1: Redact Emails by RegEx (vertebrate study)"
$section: Section(containsString("@"))
then
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
.forEach(emailEntity -> {
emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(emailEntity);
});
.forEach(emailEntity -> emailEntity.apply("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -576,10 +538,7 @@ rule "PII.4.0: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.4.1: Redact line after contact information keywords (non vertebrate study)"
@ -607,10 +566,7 @@ rule "PII.4.1: Redact line after contact information keywords (non vertebrate st
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> {
contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
@ -624,10 +580,7 @@ rule "PII.6.0: redact line between contact keywords (non vertebrate study)"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.1: redact line between contact keywords"
@ -639,10 +592,7 @@ rule "PII.6.1: redact line between contact keywords"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> {
contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(contactEntity);
});
.forEach(contactEntity -> contactEntity.apply("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -662,10 +612,7 @@ rule "PII.7.0: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
@ -683,10 +630,7 @@ rule "PII.7.1: Redact contact information if applicant is found (non vertebrate
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -706,10 +650,7 @@ rule "PII.8.0: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
insert(entity);
});
.forEach(entity -> entity.apply("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.8.1: Redact contact information if producer is found"
@ -727,10 +668,7 @@ rule "PII.8.1: Redact contact information if producer is found"
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> {
entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(entity);
});
.forEach(entity -> entity.apply("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -741,10 +679,7 @@ rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non vertebrate study)"
@ -753,10 +688,7 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (non v
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebrate study)"
@ -765,10 +697,7 @@ rule "PII.9.2: Redact between \"AUTHOR(S)\" and \"COMPLETION DATE\" (non vertebr
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.2", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (vertebrate study)"
@ -777,10 +706,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"STUDY COMPLETION DATE\" (verte
$section: Section(!hasTables(), containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
then
entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.9.3", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
@ -790,10 +716,7 @@ rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> {
authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
insert(authorEntity);
});
.forEach(authorEntity -> authorEntity.apply("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
@ -803,10 +726,7 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
$entityToExpand: RedactionEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> {
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
insert(expandedEntity);
});
.ifPresent(expandedEntity -> expandedEntity.setMatchedRuleList($entityToExpand.getMatchedRuleList()));
end
@ -831,7 +751,7 @@ rule "ETC.2.0: Redact signatures (non vertebrate study)"
$signature.apply("ETC.2.0", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
end
rule "ETC.2.0: Redact signatures (vertebrate study)"
rule "ETC.2.1: Redact signatures (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value == "Yes")
$signature: Image(imageType == ImageType.SIGNATURE)
@ -873,7 +793,7 @@ rule "ETC.5.0: Ignore dossier_redaction entries if confidentiality is not 'confi
not FileAttribute(label == "Confidentiality", value == "confidential")
$dossierRedaction: RedactionEntity(type == "dossier_redaction")
then
$dossierRedaction.setIgnored(true);
$dossierRedaction.ignore("ETC.5.0", "Ignore dossier redactions, when not confidential");
update($dossierRedaction);
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
end
@ -888,10 +808,7 @@ rule "ETC.6.0: Redact CAS Number"
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> {
redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)");
insert(redactionEntity);
});
.forEach(redactionEntity -> redactionEntity.apply("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
end
@ -934,8 +851,7 @@ rule "AI.0.0: add all NER Entities of type CBI_author"
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
then
nerEntities.streamEntitiesOfType("CBI_author")
.map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
.forEach(entity -> insert(entity));
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
end
@ -947,10 +863,7 @@ rule "AI.1.0: combine and add NER Entities as CBI_address"
then
nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
.map(boundary -> entityCreationService.forceByBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
.forEach(entity -> {
entity.addEngine(Engine.NER);
insert(entity);
});
.forEach(entity -> entity.addEngine(Engine.NER));
end
@ -969,6 +882,18 @@ rule "MAN.0.0: Apply manual resize redaction"
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId)
$imageToBeResized: Image(id == $id)
then
manualRedactionApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
@ -978,7 +903,7 @@ rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.setIgnored(true);
$entityToBeRemoved.ignore("MAN.1.0", "Removed by ManualRedaction");
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
@ -991,7 +916,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.setIgnored(true);
$imageEntityToBeRemoved.ignore("MAN.1.1", "Removed by ManualRedaction");
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
@ -1000,6 +925,7 @@ rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
@ -1011,7 +937,20 @@ rule "MAN.2.0: Apply force redaction"
$entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
no-loop true
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
$imageToForce: Image(id == $id)
then
$imageToForce.force("MAN.2.0", "Forced redaction", $legalBasis);
$imageToForce.setRemoved(false);
$imageToForce.setIgnored(false);
update($imageToForce);
update($imageToForce.getParent());
end
@ -1035,10 +974,10 @@ rule "MAN.3.0: Apply image recategorization"
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
$larger: RedactionEntity($type: type, $entityType: entityType, isActive())
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$contained.remove();
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
@ -1047,15 +986,14 @@ rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$first.remove();
$second.remove();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
$second.remove("X.1.0", "merge intersecting Entities of same type");
retract($first);
retract($second);
insert(mergedEntity);
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
end
@ -1064,11 +1002,11 @@ rule "X.1.0: merge intersecting Entities of same type"
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive())
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove();
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -1077,10 +1015,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive())
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
@ -1089,11 +1027,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove();
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
@ -1102,10 +1040,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity(entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
@ -1114,11 +1052,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
salience 32
when
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove();
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
retract($lowerRank);
end
@ -1145,6 +1083,5 @@ rule "LDS.0.0: run local dictionary search"
when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
.forEach(entity -> insert(entity));
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document).toList();
end

47
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -80,16 +80,18 @@ rule "Always redact PII"
$cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: remove Entity contained by Entity of same type"
salience 65
when
$larger: RedactionEntity($type: type, $entityType: entityType)
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
$larger: RedactionEntity($type: type, $entityType: entityType, isActive())
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$contained.remove();
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
@ -98,15 +100,14 @@ rule "X.0.0: remove Entity contained by Entity of same type"
rule "X.1.0: merge intersecting Entities of same type"
salience 64
when
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$first.remove();
$second.remove();
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
$second.remove("X.1.0", "merge intersecting Entities of same type");
retract($first);
retract($second);
insert(mergedEntity);
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
end
@ -115,11 +116,11 @@ rule "X.1.0: merge intersecting Entities of same type"
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE, isActive())
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove();
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
@ -128,10 +129,10 @@ rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, isActive())
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
@ -140,11 +141,11 @@ rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMM
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove();
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
@ -153,10 +154,10 @@ rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY wit
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: RedactionEntity(entityType == EntityType.ENTITY)
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
$entity: RedactionEntity(entityType == EntityType.ENTITY, isActive())
$recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$recommendation.remove();
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
@ -165,11 +166,11 @@ rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
salience 32
when
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY)
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger)
$higherRank: RedactionEntity($type: type, entityType == EntityType.ENTITY, isActive())
$lowerRank: RedactionEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized, !skipRemoveEntitiesContainedInLarger, isActive())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove();
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
retract($lowerRank);
end

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/log4j2-test.xml
View File

@ -10,7 +10,7 @@
<Root level="warn">
<AppenderRef ref="CONSOLE"/>
</Root>
<Logger name="com.iqser" level="debug"/>
<Logger name="com.iqser" level="info"/>
</Loggers>
</Configuration>

877
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

File diff suppressed because it is too large Load Diff