RED-6368: Include Tables and Images into Prototype Document Structure
*started Implementation of new rules
This commit is contained in:
Kilian Schuettler
parent
f73b8fe64e
commit
8f38576d10
@ -10,34 +10,141 @@ import java.util.HashSet;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.*
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
||||
import java.util.Set;
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||
import java.util.Set
|
||||
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
||||
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
|
||||
|
||||
global DocumentGraph document
|
||||
global EntityCreationService entityCreationService
|
||||
|
||||
|
||||
rule "1: Redact CBI_author"
|
||||
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||||
no-loop true
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
entity: EntityNode(type == "CBI_author")
|
||||
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||||
then
|
||||
entity.setRedaction(true);
|
||||
update(entity)
|
||||
$entity.setRedaction(true);
|
||||
if ($entity.getMatchedRule() == -1) {
|
||||
$entity.setMatchedRule(1);
|
||||
}
|
||||
if ($entity.getRedactionReason().equals("")) {
|
||||
$entity.setRedactionReason("Author found");
|
||||
}
|
||||
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
$entity.addEngine(Engine.RULE);
|
||||
update($entity)
|
||||
end
|
||||
|
||||
rule "2: do not redact genitive CBI_author"
|
||||
rule "2: Redact CBI Authors (Vertebrate study)"
|
||||
no-loop true
|
||||
|
||||
when
|
||||
entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||||
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||||
then
|
||||
entity.setRedaction(false);
|
||||
entity.setEntityType(EntityType.FALSE_POSITIVE);
|
||||
update(entity)
|
||||
$entity.setRedaction(true);
|
||||
if ($entity.getMatchedRule() == -1) {
|
||||
$entity.setMatchedRule(2);
|
||||
}
|
||||
if ($entity.getRedactionReason().equals("")) {
|
||||
$entity.setRedactionReason("Author found");
|
||||
}
|
||||
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||
$entity.addEngine(Engine.RULE);
|
||||
update($entity)
|
||||
end
|
||||
|
||||
rule "3: Don't redact CBI Address (Non vertebrate study)"
|
||||
no-loop true
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||||
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||||
$recommendationEntity: EntityNode(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
|
||||
then
|
||||
$entity.setRedaction(false);
|
||||
$entity.setMatchedRule(3);
|
||||
$entity.setRedactionReason("Address found for non vertebrate study");
|
||||
$entity.addEngine(Engine.RULE);
|
||||
update($entity)
|
||||
$recommendationEntity.removeFromGraph();
|
||||
delete($recommendationEntity);
|
||||
end
|
||||
|
||||
rule "4: Redact CBI Address (Vertebrate study)"
|
||||
no-loop true
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||||
then
|
||||
$entity.setRedaction(true);
|
||||
$entity.setMatchedRule(4);
|
||||
$entity.setRedactionReason("Address found");
|
||||
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||
$entity.addEngine(Engine.RULE);
|
||||
update($entity)
|
||||
end
|
||||
|
||||
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
|
||||
|
||||
when
|
||||
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
||||
then
|
||||
EntityNode entity = entityCreationService.createEntityByEntity($entity, "CBI_author", EntityType.FALSE_POSITIVE, document);
|
||||
entity.setMatchedRule(5);
|
||||
insert(entity);
|
||||
end
|
||||
|
||||
|
||||
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header"
|
||||
|
||||
when
|
||||
authorCell: TableCellNode(header == false, (hasHeader("Author(s)") || hasHeader("Author")))
|
||||
then
|
||||
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY, document);
|
||||
entity.setMatchedRule(6);
|
||||
entity.setRedactionReason("Header \"Author(s)\" found");
|
||||
insert(entity);
|
||||
end
|
||||
|
||||
rule "7: Add CBI_author with \"et al.\" Regex"
|
||||
|
||||
when
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, document);
|
||||
entities.forEach(entity -> entity.setMatchedRule(7));
|
||||
entities.forEach(entity -> entity.setRedactionReason("Found by \"et al.\" regex"));
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
rule "8: Add recommendation for Addresses in Test Organism sections"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode(containsString("Species") && containsString("Source"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", $section, "CBI_address", EntityType.RECOMMENDATION, document);
|
||||
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source\" in Test Organism Section"));
|
||||
entities.forEach(entity -> entity.setMatchedRule(8));
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
|
||||
rule "9: Add recommendation for Addresses in Test Animals sections"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode(containsString("Species:") && containsString("Source:"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", $section, "CBI_address", EntityType.RECOMMENDATION, document);
|
||||
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source:\" in Test Organism Section"));
|
||||
entities.forEach(entity -> entity.setMatchedRule(9));
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
|
||||
@ -17,40 +17,66 @@ import java.util.Set;
|
||||
|
||||
global DocumentGraph document
|
||||
|
||||
rule "remove contained entity"
|
||||
|
||||
rule "merge contained Entities of same type"
|
||||
salience 100
|
||||
when
|
||||
$largerEntity: EntityNode()
|
||||
$smallerEntity: EntityNode(this != $largerEntity, containedBy($largerEntity))
|
||||
outer: EntityNode($type: type, $entityType: entityType)
|
||||
inner: EntityNode(this != outer, containedBy(outer), type == $type, entityType == $entityType)
|
||||
then
|
||||
$smallerEntity.removeFromGraph();
|
||||
$smallerEntity.setRemoved(true);
|
||||
$smallerEntity.setSkipRemoveEntitiesContainedInLarger(true);
|
||||
System.out.printf("%s contained by %s\n", $smallerEntity, $largerEntity);
|
||||
delete($smallerEntity);
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
rule "1: Redact CBI_author"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
entity: EntityNode(type == "CBI_author")
|
||||
then
|
||||
entity.setRedaction(true);
|
||||
update(entity)
|
||||
System.out.printf("removed entity %s contained by %s\n", inner, outer);
|
||||
outer.addEngines(inner.getEngines());
|
||||
inner.removeFromGraph();
|
||||
delete(inner);
|
||||
end
|
||||
|
||||
rule "2: do not redact genitive CBI_author"
|
||||
|
||||
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||
salience 100
|
||||
when
|
||||
entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
||||
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
|
||||
entity: EntityNode(this != $falsePositive, containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY)
|
||||
then
|
||||
entity.setRedaction(false);
|
||||
entity.setEntityType(EntityType.FALSE_POSITIVE);
|
||||
update(entity)
|
||||
System.out.printf("removed entity %s marked false positive by %s\n", entity, $falsePositive);
|
||||
entity.removeFromGraph();
|
||||
delete(entity);
|
||||
end
|
||||
|
||||
/*
|
||||
rule "If same type, remove inner"
|
||||
salience 100
|
||||
when
|
||||
$containedEntity: ContainedEntity($toRemove: inner, outer.getType() == inner.getType())
|
||||
then
|
||||
$toRemove.removeFromGraph();
|
||||
delete($toRemove);
|
||||
delete($containedEntity);
|
||||
end
|
||||
|
||||
rule "If outer Recommended and inner Entity, remove outer"
|
||||
salience 99
|
||||
when
|
||||
$containedEntity: ContainedEntity($toRemove: outer, outer.getType() == EntityType.RECOMMENDATION, inner.getType() == EntityType.ENTITY)
|
||||
then
|
||||
$toRemove.removeFromGraph();
|
||||
delete($toRemove);
|
||||
delete($containedEntity);
|
||||
end
|
||||
|
||||
rule "Inner not False Recommendation and outer not Entity"
|
||||
salience 99
|
||||
when
|
||||
$containedEntity: ContainedEntity($toRemove: outer, (outer.getType() != EntityType.ENTITY || inner.getType() != EntityType.FALSE_RECOMMENDATION))
|
||||
then
|
||||
$toRemove.removeFromGraph();
|
||||
delete($toRemove);
|
||||
delete($containedEntity);
|
||||
end
|
||||
rule "Remove leftover ContainedEntities"
|
||||
salience 0
|
||||
when
|
||||
$toRemove: ContainedEntity()
|
||||
then
|
||||
delete($toRemove);
|
||||
end
|
||||
|
||||
*/
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user