9232973a00
* refactoring structure * added custom syntax validation, such that rule files may not be corrupted * added component drools preparations * deprecated redactionLog * integrated entityLog Signed-off-by: Kilian Schuettler <kilian.schuettler@knecon.com>
352 lines
16 KiB
Plaintext
352 lines
16 KiB
Plaintext
package drools
|
|
|
|
import static java.lang.String.format;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
|
|
|
|
import java.util.List;
|
|
import java.util.LinkedList;
|
|
import java.util.Set;
|
|
import java.util.stream.Collectors;
|
|
import java.util.Collection;
|
|
import java.util.stream.Stream;
|
|
import java.util.Optional;
|
|
|
|
import com.iqser.red.service.redaction.v1.server.model.document.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
|
import com.iqser.red.service.redaction.v1.server.service.graph.EntityCreationService;
|
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
|
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
|
|
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
|
|
|
|
global Document document
|
|
global EntityCreationService entityCreationService
|
|
global ManualChangesApplicationService manualChangesApplicationService
|
|
global Dictionary dictionary
|
|
// --------------------------------------- queries -------------------------------------------------------------------
|
|
|
|
query "getFileAttributes"
|
|
$fileAttribute: FileAttribute()
|
|
end
|
|
|
|
// --------------------------------------- NER Entities rules -------------------------------------------------------------------
|
|
|
|
// Rule unit: AI.0
|
|
rule "AI.0.0: add all NER Entities of type CBI_author"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("CBI_author")
|
|
.forEach(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
|
|
end
|
|
|
|
// --------------------------------------- CBI rules -------------------------------------------------------------------
|
|
|
|
rule "CBI.0.0: Always redact CBI_author"
|
|
|
|
when
|
|
$cbiAuthor: TextEntity(type == "CBI_author", entityType == EntityType.ENTITY)
|
|
then
|
|
$cbiAuthor.apply("CBI.0.0", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
// --------------------------------------- PII rules -------------------------------------------------------------------
|
|
|
|
rule "PII.0.0: Always redact PII"
|
|
|
|
when
|
|
$cbiAuthor: TextEntity(type == "PII", entityType == EntityType.ENTITY)
|
|
then
|
|
$cbiAuthor.apply("PII.0.0", "PII found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
|
|
//------------------------------------ Manual redaction rules ------------------------------------
|
|
|
|
// Rule unit: MAN.0
|
|
rule "MAN.0.0: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$entityToBeResized: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
update($entityToBeResized);
|
|
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "MAN.0.1: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$imageToBeResized: Image(id == $id)
|
|
then
|
|
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
update($imageToBeResized);
|
|
update($imageToBeResized.getParent());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.1
|
|
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
|
update($entityToBeRemoved);
|
|
retract($idRemoval);
|
|
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$imageEntityToBeRemoved: Image($id == id)
|
|
then
|
|
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
|
update($imageEntityToBeRemoved);
|
|
retract($idRemoval);
|
|
update($imageEntityToBeRemoved.getParent());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.2
|
|
rule "MAN.2.0: Apply force redaction"
|
|
no-loop true
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$entityToForce: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToForce.getManualOverwrite().addChange($force);
|
|
update($entityToForce);
|
|
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "MAN.2.1: Apply force redaction to images"
|
|
no-loop true
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$imageToForce: Image(id == $id)
|
|
then
|
|
$imageToForce.getManualOverwrite().addChange($force);
|
|
update($imageToForce);
|
|
update($imageToForce.getParent());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.3
|
|
rule "MAN.3.0: Apply entity recategorization"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
|
|
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
|
|
retract($recategorization);
|
|
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
|
|
retract($entityToBeRecategorized);
|
|
end
|
|
|
|
rule "MAN.3.1: Apply image recategorization"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$imageToBeRecategorized: Image($id == id)
|
|
then
|
|
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
|
|
update($imageToBeRecategorized);
|
|
update($imageToBeRecategorized.getParent());
|
|
retract($recategorization);
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.4
|
|
rule "MAN.4.0: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$imageToBeRecategorized: Image($id == id)
|
|
then
|
|
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
|
|
end
|
|
|
|
rule "MAN.4.1: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
|
|
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
|
|
end
|
|
|
|
|
|
//------------------------------------ Entity merging rules ------------------------------------
|
|
|
|
// Rule unit: X.0
|
|
rule "X.0.0: remove Entity contained by Entity of same type"
|
|
salience 65
|
|
when
|
|
$larger: TextEntity($type: type, $entityType: entityType, active())
|
|
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized(), active())
|
|
then
|
|
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
|
|
retract($contained);
|
|
end
|
|
|
|
|
|
// Rule unit: X.1
|
|
rule "X.1.0: merge intersecting Entities of same type"
|
|
salience 64
|
|
when
|
|
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
|
|
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized(), active())
|
|
then
|
|
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
|
|
$first.remove("X.1.0", "merge intersecting Entities of same type");
|
|
$second.remove("X.1.0", "merge intersecting Entities of same type");
|
|
retract($first);
|
|
retract($second);
|
|
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
|
|
// Rule unit: X.2
|
|
rule "X.2.0: remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
|
|
$entity: TextEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized(), active())
|
|
then
|
|
$entity.getIntersectingNodes().forEach(node -> update(node));
|
|
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
|
|
retract($entity)
|
|
end
|
|
|
|
|
|
// Rule unit: X.3
|
|
rule "X.3.0: remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
|
salience 64
|
|
when
|
|
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
|
|
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
|
|
then
|
|
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.4
|
|
rule "X.4.0: remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity($type: type, entityType == EntityType.ENTITY, active())
|
|
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized(), active())
|
|
then
|
|
$entity.addEngines($recommendation.getEngines());
|
|
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.5
|
|
rule "X.5.0: remove Entity of type RECOMMENDATION when contained by ENTITY"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity(entityType == EntityType.ENTITY, active())
|
|
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized(), active())
|
|
then
|
|
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.6
|
|
rule "X.6.0: remove Entity of lower rank, when intersected by entity of type ENTITY"
|
|
salience 32
|
|
when
|
|
$higherRank: TextEntity($type: type, entityType == EntityType.ENTITY, active())
|
|
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !resized(), active())
|
|
then
|
|
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
|
|
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when intersected by entity of type ENTITY");
|
|
retract($lowerRank);
|
|
end
|
|
|
|
|
|
//------------------------------------ File attributes rules ------------------------------------
|
|
|
|
// Rule unit: FA.1
|
|
rule "FA.1.0: remove duplicate FileAttributes"
|
|
salience 64
|
|
when
|
|
$fileAttribute: FileAttribute($label: label, $value: value)
|
|
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
|
|
then
|
|
retract($duplicate);
|
|
end
|
|
|
|
|
|
//------------------------------------ Local dictionary search rules ------------------------------------
|
|
|
|
// Rule unit: LDS.0
|
|
rule "LDS.0.0: run local dictionary search"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -999
|
|
when
|
|
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
|
|
then
|
|
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
|
|
.forEach(entity -> {
|
|
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
|
|
entity.addMatchedRules(matchedRules);
|
|
});
|
|
end
|