179 lines
8.7 KiB
Plaintext
179 lines
8.7 KiB
Plaintext
package drools
|
|
|
|
import static java.lang.String.format;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.anyMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils.exactMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
|
|
|
|
import java.util.List;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
|
|
import java.util.LinkedList;
|
|
import java.util.HashSet;
|
|
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
|
import java.util.Set
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
|
|
import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
|
|
import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary
|
|
import java.util.stream.Collectors
|
|
import java.util.Collection
|
|
import java.util.stream.Stream
|
|
import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtils;
|
|
|
|
global DocumentGraph document
|
|
global EntityCreationService entityCreationService
|
|
global ManualRedactionApplicationService manualRedactionApplicationService
|
|
global Dictionary dictionary
|
|
|
|
// --------------------------------------- queries -------------------------------------------------------------------
|
|
|
|
query "getFileAttributes"
|
|
$fileAttribute: FileAttribute()
|
|
end
|
|
|
|
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
|
|
|
|
rule "add NER Entities of type CBI_author or CBI_address"
|
|
salience 999
|
|
when
|
|
$nerEntity: EntityRecognitionEntity($type: type, (type == "CBI_author" || type == "CBI_address"))
|
|
then
|
|
RedactionEntity redactionEntity = entityCreationService.byBoundary(new Boundary($nerEntity.getStartOffset(), $nerEntity.getEndOffset()), $type, EntityType.RECOMMENDATION, document);
|
|
redactionEntity.addEngine(Engine.NER);
|
|
insert(redactionEntity);
|
|
end
|
|
|
|
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
|
|
|
rule "Always redact CBI_author"
|
|
|
|
when
|
|
$cbiAuthor: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
|
|
then
|
|
$cbiAuthor.addMatchedRule(0);
|
|
$cbiAuthor.setRedaction(true);
|
|
$cbiAuthor.setRedactionReason("Author found");
|
|
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
// --------------------------------------- PII rules -------------------------------------------------------------------
|
|
|
|
rule "Always redact PII"
|
|
|
|
when
|
|
$cbiAuthor: RedactionEntity(type == "PII", entityType == EntityType.ENTITY)
|
|
then
|
|
$cbiAuthor.addMatchedRule(1);
|
|
$cbiAuthor.setRedaction(true);
|
|
$cbiAuthor.setRedactionReason("PII found");
|
|
$cbiAuthor.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
// --------------------------------------- merging rules -------------------------------------------------------------------
|
|
|
|
rule "remove Entity contained by Entity of same type"
|
|
salience 65
|
|
when
|
|
$larger: RedactionEntity($type: type, $entityType: entityType)
|
|
$contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$contained.removeFromGraph();
|
|
retract($contained);
|
|
end
|
|
|
|
rule "merge intersecting Entities of same type"
|
|
salience 64
|
|
when
|
|
$first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
$second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$first.removeFromGraph();
|
|
$second.removeFromGraph();
|
|
RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
|
|
retract($first);
|
|
retract($second);
|
|
insert(mergedEntity);
|
|
end
|
|
|
|
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
|
|
$entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$entity.removeFromGraph();
|
|
retract($entity)
|
|
end
|
|
|
|
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
|
salience 64
|
|
when
|
|
$falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
|
|
$recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$recommendation.removeFromGraph();
|
|
retract($recommendation);
|
|
end
|
|
|
|
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
|
salience 64
|
|
when
|
|
$entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
|
|
$recommendation: RedactionEntity(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
|
then
|
|
$recommendation.removeFromGraph();
|
|
retract($recommendation);
|
|
end
|
|
|
|
rule "remove Entity of lower rank, when equal boundaries and entityType"
|
|
salience 32
|
|
when
|
|
$higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
|
|
$lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
|
|
then
|
|
$lowerRank.removeFromGraph();
|
|
retract($lowerRank);
|
|
end
|
|
|
|
// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
|
|
|
|
rule "remove duplicate FileAttributes"
|
|
salience 64
|
|
when
|
|
$first: FileAttribute($label: label, $value: value)
|
|
$second: FileAttribute(this != $first, label == $label, value == $value)
|
|
then
|
|
retract($second);
|
|
end
|
|
|
|
// --------------------------------------- local dictionary search -------------------------------------------------------------------
|
|
|
|
rule "run local dictionary search"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -999
|
|
when
|
|
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
|
|
then
|
|
entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
|
|
.forEach(redactionEntity -> insert(redactionEntity));
|
|
end
|