RED-6368: Include Tables and Images into Prototype Document Structure
*started Implementation of new rules
This commit is contained in:
Kilian Schuettler
Kilian Schuettler
parent
d04a576397
commit
253e544724
@ -10,34 +10,141 @@ import java.util.HashSet;
|
|||||||
|
|
||||||
import com.iqser.red.service.redaction.v1.server.document.graph.*
|
import com.iqser.red.service.redaction.v1.server.document.graph.*
|
||||||
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
|
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
|
||||||
|
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
|
||||||
|
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
||||||
import com.iqser.red.service.redaction.v1.model.FileAttribute;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
||||||
import com.iqser.red.service.redaction.v1.model.Engine;
|
import java.util.Set
|
||||||
import com.iqser.red.service.persistence.service.v1.api.model.dossiertemplate.type.DictionaryEntry;
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
||||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.EntitySearchUtils;
|
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
global DocumentGraph document
|
global DocumentGraph document
|
||||||
|
global EntityCreationService entityCreationService
|
||||||
|
|
||||||
|
rule "1: Redact CBI Authors (Non vertebrate study)"
|
||||||
rule "1: Redact CBI_author"
|
no-loop true
|
||||||
|
|
||||||
when
|
when
|
||||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||||
entity: EntityNode(type == "CBI_author")
|
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||||||
then
|
then
|
||||||
entity.setRedaction(true);
|
$entity.setRedaction(true);
|
||||||
update(entity)
|
if ($entity.getMatchedRule() == -1) {
|
||||||
|
$entity.setMatchedRule(1);
|
||||||
|
}
|
||||||
|
if ($entity.getRedactionReason().equals("")) {
|
||||||
|
$entity.setRedactionReason("Author found");
|
||||||
|
}
|
||||||
|
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
$entity.addEngine(Engine.RULE);
|
||||||
|
update($entity)
|
||||||
end
|
end
|
||||||
|
|
||||||
rule "2: do not redact genitive CBI_author"
|
rule "2: Redact CBI Authors (Vertebrate study)"
|
||||||
|
no-loop true
|
||||||
|
|
||||||
when
|
when
|
||||||
entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||||||
|
$entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
|
||||||
then
|
then
|
||||||
entity.setRedaction(false);
|
$entity.setRedaction(true);
|
||||||
entity.setEntityType(EntityType.FALSE_POSITIVE);
|
if ($entity.getMatchedRule() == -1) {
|
||||||
update(entity)
|
$entity.setMatchedRule(2);
|
||||||
|
}
|
||||||
|
if ($entity.getRedactionReason().equals("")) {
|
||||||
|
$entity.setRedactionReason("Author found");
|
||||||
|
}
|
||||||
|
$entity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
|
||||||
|
$entity.addEngine(Engine.RULE);
|
||||||
|
update($entity)
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "3: Don't redact CBI Address (Non vertebrate study)"
|
||||||
|
no-loop true
|
||||||
|
|
||||||
|
when
|
||||||
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
|
||||||
|
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||||||
|
$recommendationEntity: EntityNode(type == "CBI_address", entityType == EntityType.RECOMMENDATION)
|
||||||
|
then
|
||||||
|
$entity.setRedaction(false);
|
||||||
|
$entity.setMatchedRule(3);
|
||||||
|
$entity.setRedactionReason("Address found for non vertebrate study");
|
||||||
|
$entity.addEngine(Engine.RULE);
|
||||||
|
update($entity)
|
||||||
|
$recommendationEntity.removeFromGraph();
|
||||||
|
delete($recommendationEntity);
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "4: Redact CBI Address (Vertebrate study)"
|
||||||
|
no-loop true
|
||||||
|
|
||||||
|
when
|
||||||
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||||
|
$entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
|
||||||
|
then
|
||||||
|
$entity.setRedaction(true);
|
||||||
|
$entity.setMatchedRule(4);
|
||||||
|
$entity.setRedactionReason("Address found");
|
||||||
|
$entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
|
||||||
|
$entity.addEngine(Engine.RULE);
|
||||||
|
update($entity)
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
|
||||||
|
|
||||||
|
when
|
||||||
|
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
||||||
|
then
|
||||||
|
EntityNode entity = entityCreationService.createEntityByEntity($entity, "CBI_author", EntityType.FALSE_POSITIVE, document);
|
||||||
|
entity.setMatchedRule(5);
|
||||||
|
insert(entity);
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
|
rule "6: Create Entity from Author(s) cells in Tables with Author(s) header"
|
||||||
|
|
||||||
|
when
|
||||||
|
authorCell: TableCellNode(header == false, (hasHeader("Author(s)") || hasHeader("Author")))
|
||||||
|
then
|
||||||
|
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY, document);
|
||||||
|
entity.setMatchedRule(6);
|
||||||
|
entity.setRedactionReason("Header \"Author(s)\" found");
|
||||||
|
insert(entity);
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "7: Add CBI_author with \"et al.\" Regex"
|
||||||
|
|
||||||
|
when
|
||||||
|
then
|
||||||
|
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, document);
|
||||||
|
entities.forEach(entity -> entity.setMatchedRule(7));
|
||||||
|
entities.forEach(entity -> entity.setRedactionReason("Found by \"et al.\" regex"));
|
||||||
|
entities.forEach(entity -> insert(entity));
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "8: Add recommendation for Addresses in Test Organism sections"
|
||||||
|
|
||||||
|
when
|
||||||
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||||
|
$section: SectionNode(containsString("Species") && containsString("Source"))
|
||||||
|
then
|
||||||
|
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", $section, "CBI_address", EntityType.RECOMMENDATION, document);
|
||||||
|
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source\" in Test Organism Section"));
|
||||||
|
entities.forEach(entity -> entity.setMatchedRule(8));
|
||||||
|
entities.forEach(entity -> insert(entity));
|
||||||
|
end
|
||||||
|
|
||||||
|
|
||||||
|
rule "9: Add recommendation for Addresses in Test Animals sections"
|
||||||
|
|
||||||
|
when
|
||||||
|
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||||
|
$section: SectionNode(containsString("Species:") && containsString("Source:"))
|
||||||
|
then
|
||||||
|
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", $section, "CBI_address", EntityType.RECOMMENDATION, document);
|
||||||
|
entities.forEach(entity -> entity.setRedactionReason("Line after \"Source:\" in Test Organism Section"));
|
||||||
|
entities.forEach(entity -> entity.setMatchedRule(9));
|
||||||
|
entities.forEach(entity -> insert(entity));
|
||||||
|
end
|
||||||
|
|
||||||
|
|||||||
@ -17,40 +17,66 @@ import java.util.Set;
|
|||||||
|
|
||||||
global DocumentGraph document
|
global DocumentGraph document
|
||||||
|
|
||||||
rule "remove contained entity"
|
|
||||||
|
|
||||||
|
rule "merge contained Entities of same type"
|
||||||
|
salience 100
|
||||||
when
|
when
|
||||||
$largerEntity: EntityNode()
|
outer: EntityNode($type: type, $entityType: entityType)
|
||||||
$smallerEntity: EntityNode(this != $largerEntity, containedBy($largerEntity))
|
inner: EntityNode(this != outer, containedBy(outer), type == $type, entityType == $entityType)
|
||||||
then
|
then
|
||||||
$smallerEntity.removeFromGraph();
|
System.out.printf("removed entity %s contained by %s\n", inner, outer);
|
||||||
$smallerEntity.setRemoved(true);
|
outer.addEngines(inner.getEngines());
|
||||||
$smallerEntity.setSkipRemoveEntitiesContainedInLarger(true);
|
inner.removeFromGraph();
|
||||||
System.out.printf("%s contained by %s\n", $smallerEntity, $largerEntity);
|
delete(inner);
|
||||||
delete($smallerEntity);
|
|
||||||
end
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
rule "1: Redact CBI_author"
|
|
||||||
|
|
||||||
when
|
|
||||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
|
||||||
entity: EntityNode(type == "CBI_author")
|
|
||||||
then
|
|
||||||
entity.setRedaction(true);
|
|
||||||
update(entity)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
rule "2: do not redact genitive CBI_author"
|
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||||
|
salience 100
|
||||||
when
|
when
|
||||||
entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction == true)
|
$falsePositive: EntityNode($type: type, entityType == EntityType.FALSE_POSITIVE)
|
||||||
|
entity: EntityNode(this != $falsePositive, containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY)
|
||||||
then
|
then
|
||||||
entity.setRedaction(false);
|
System.out.printf("removed entity %s marked false positive by %s\n", entity, $falsePositive);
|
||||||
entity.setEntityType(EntityType.FALSE_POSITIVE);
|
entity.removeFromGraph();
|
||||||
update(entity)
|
delete(entity);
|
||||||
end
|
end
|
||||||
|
|
||||||
|
/*
|
||||||
|
rule "If same type, remove inner"
|
||||||
|
salience 100
|
||||||
|
when
|
||||||
|
$containedEntity: ContainedEntity($toRemove: inner, outer.getType() == inner.getType())
|
||||||
|
then
|
||||||
|
$toRemove.removeFromGraph();
|
||||||
|
delete($toRemove);
|
||||||
|
delete($containedEntity);
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "If outer Recommended and inner Entity, remove outer"
|
||||||
|
salience 99
|
||||||
|
when
|
||||||
|
$containedEntity: ContainedEntity($toRemove: outer, outer.getType() == EntityType.RECOMMENDATION, inner.getType() == EntityType.ENTITY)
|
||||||
|
then
|
||||||
|
$toRemove.removeFromGraph();
|
||||||
|
delete($toRemove);
|
||||||
|
delete($containedEntity);
|
||||||
|
end
|
||||||
|
|
||||||
|
rule "Inner not False Recommendation and outer not Entity"
|
||||||
|
salience 99
|
||||||
|
when
|
||||||
|
$containedEntity: ContainedEntity($toRemove: outer, (outer.getType() != EntityType.ENTITY || inner.getType() != EntityType.FALSE_RECOMMENDATION))
|
||||||
|
then
|
||||||
|
$toRemove.removeFromGraph();
|
||||||
|
delete($toRemove);
|
||||||
|
delete($containedEntity);
|
||||||
|
end
|
||||||
|
rule "Remove leftover ContainedEntities"
|
||||||
|
salience 0
|
||||||
|
when
|
||||||
|
$toRemove: ContainedEntity()
|
||||||
|
then
|
||||||
|
delete($toRemove);
|
||||||
|
end
|
||||||
|
|
||||||
|
*/
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user