660 lines
26 KiB
Plaintext
660 lines
26 KiB
Plaintext
package drools
|
|
|
|
import static java.lang.String.format;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
|
|
|
|
import java.util.List;
|
|
import java.util.LinkedList;
|
|
import java.util.Set;
|
|
import java.util.stream.Collectors;
|
|
import java.util.Collection;
|
|
import java.util.stream.Stream;
|
|
import java.util.Optional;
|
|
|
|
import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
|
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
|
|
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
|
|
|
global Document document
|
|
global EntityCreationService entityCreationService
|
|
global ManualChangesApplicationService manualChangesApplicationService
|
|
global Dictionary dictionary
|
|
global RulesLogger logger
|
|
|
|
//------------------------------------ queries ------------------------------------
|
|
|
|
query "getFileAttributes"
|
|
$fileAttribute: FileAttribute()
|
|
end
|
|
|
|
//------------------------------------ CBI rules ------------------------------------
|
|
|
|
// Rule unit: CBI.0
|
|
rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
|
|
then
|
|
$entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3");
|
|
end
|
|
|
|
rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$entity: TextEntity(type() == "CBI_author", dictionaryEntry)
|
|
then
|
|
$entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2");
|
|
end
|
|
|
|
|
|
//------------------------------------ PII rules ------------------------------------
|
|
|
|
// Rule unit: PII.0
|
|
rule "PII.0.1: Redact all PII (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$pii: TextEntity(type() == "PII", dictionaryEntry)
|
|
then
|
|
$pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3");
|
|
end
|
|
|
|
|
|
//------------------------------------ AI rules ------------------------------------
|
|
|
|
// Rule unit: AI.0
|
|
rule "AI.0.0: Add all NER Entities of type CBI_author"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("CBI_author")
|
|
.filter(entity -> entity.value().length() > 3)
|
|
.filter(entity -> entity.value().length() < 40)
|
|
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
|
|
end
|
|
|
|
|
|
// Rule unit: AI.4
|
|
rule "AI.4.0: Add all NER Entities of type Person"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("Person"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("Person")
|
|
.filter(entity -> entity.value().length() > 3)
|
|
.filter(entity -> entity.value().length() < 100)
|
|
.forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_author", EntityType.RECOMMENDATION, document)
|
|
.ifPresent(e -> e.skip("AI.4.0", "")));
|
|
end
|
|
|
|
|
|
// Rule unit: AI.5
|
|
rule "AI.5.0: Combine and add NER Entities as CBI_address"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("Location") || hasEntitiesOfType("Address")|| hasEntitiesOfType("Organization"))
|
|
then
|
|
entityCreationService
|
|
.combineNerEntitiesWithConfidence(
|
|
nerEntities,
|
|
"CBI_address",
|
|
EntityType.RECOMMENDATION,
|
|
document,
|
|
Set.of("Organization",
|
|
"Location",
|
|
"Address",
|
|
"ORG",
|
|
"STREET",
|
|
"CITY"),
|
|
Set.of("Organization",
|
|
"Location",
|
|
"Address",
|
|
"ORG",
|
|
"STREET",
|
|
"POSTAL",
|
|
"COUNTRY",
|
|
"CARDINAL",
|
|
"CITY",
|
|
"STATE"),
|
|
50,
|
|
3,
|
|
2,
|
|
0.7)
|
|
.forEach(nerEntity -> nerEntity.skip("AI.5.0", ""));
|
|
end
|
|
|
|
|
|
// Rule unit: AI.6
|
|
rule "AI.6.0: Add all NER Entities of type Location"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("Location"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("Location")
|
|
.filter(entity -> entity.value().length() > 3)
|
|
.filter(entity -> entity.value().length() < 100)
|
|
.forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
|
|
.ifPresent(e -> e.skip("AI.6.0", "")));
|
|
end
|
|
|
|
|
|
// Rule unit: AI.7
|
|
rule "AI.7.0: Add all NER Entities of type Address"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("Address"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("Address")
|
|
.filter(entity -> entity.value().length() > 3)
|
|
.filter(entity -> entity.value().length() < 100)
|
|
.forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
|
|
.ifPresent(e -> e.skip("AI.7.0", "")));
|
|
end
|
|
|
|
|
|
//------------------------------------ Manual changes rules ------------------------------------
|
|
|
|
// Rule unit: MAN.0
|
|
rule "MAN.0.0: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$entityToBeResized: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
end
|
|
|
|
rule "MAN.0.1: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$imageToBeResized: Image(id == $id)
|
|
then
|
|
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.1
|
|
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
|
|
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeRemoved.addManualChange($idRemoval);
|
|
retract($idRemoval);
|
|
end
|
|
|
|
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId)
|
|
$imageEntityToBeRemoved: Image($id == id)
|
|
then
|
|
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
|
retract($idRemoval);
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.2
|
|
rule "MAN.2.0: Apply force redaction"
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId)
|
|
$entityToForce: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToForce.addManualChange($force);
|
|
retract($force);
|
|
end
|
|
|
|
rule "MAN.2.1: Apply force redaction to images"
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId)
|
|
$imageToForce: Image(id == $id)
|
|
then
|
|
$imageToForce.getManualOverwrite().addChange($force);
|
|
retract($force);
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.3
|
|
rule "MAN.3.0: Apply entity recategorization"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
|
|
then
|
|
$entityToBeRecategorized.addManualChange($recategorization);
|
|
retract($recategorization);
|
|
end
|
|
|
|
rule "MAN.3.1: Apply entity recategorization of same type"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
|
|
then
|
|
$entityToBeRecategorized.addManualChange($recategorization);
|
|
retract($recategorization);
|
|
end
|
|
|
|
rule "MAN.3.3: Apply recategorization entities by default"
|
|
salience 128
|
|
when
|
|
$entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
|
|
then
|
|
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.4
|
|
rule "MAN.4.0: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
|
|
$imageToBeRecategorized: Image($id == id)
|
|
then
|
|
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
|
|
retract($legalBasisChange)
|
|
end
|
|
|
|
rule "MAN.4.1: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
|
|
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeChanged.addManualChange($legalBasisChange);
|
|
retract($legalBasisChange)
|
|
end
|
|
|
|
|
|
//------------------------------------ Entity merging rules ------------------------------------
|
|
|
|
// Rule unit: X.0
|
|
rule "X.0.0: Remove Entity contained by Entity of same type"
|
|
salience 65
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.type() == $contained.type(),
|
|
$container.entityType == $contained.entityType,
|
|
$container != $contained,
|
|
!$container.removed(),
|
|
!$container.hasManualChanges(),
|
|
!$contained.hasManualChanges(),
|
|
!$contained.removed()
|
|
)
|
|
not TextEntity(
|
|
getTextRange().equals($container.getTextRange()),
|
|
type() == $container.type(),
|
|
entityType == EntityType.DICTIONARY_REMOVAL,
|
|
engines contains Engine.DOSSIER_DICTIONARY,
|
|
!hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
|
|
end
|
|
|
|
rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
|
|
salience 65
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.type() == $contained.type(),
|
|
$container.entityType == $contained.entityType,
|
|
$container != $contained,
|
|
!$container.removed(),
|
|
$container.hasManualChanges(),
|
|
!$contained.hasManualChanges(),
|
|
!$contained.removed()
|
|
)
|
|
then
|
|
$contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
|
|
end
|
|
|
|
|
|
// Rule unit: X.2
|
|
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.entityType == EntityType.FALSE_POSITIVE,
|
|
$container.active(),
|
|
$contained.entityType == EntityType.ENTITY,
|
|
$contained.type() == $container.type(),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
|
|
end
|
|
|
|
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.entityType == EntityType.FALSE_POSITIVE,
|
|
$container.active(),
|
|
$contained.entityType == EntityType.HINT,
|
|
$contained.type() == $container.type(),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE");
|
|
end
|
|
|
|
|
|
rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
|
|
salience 64
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.entityType == EntityType.FALSE_RECOMMENDATION,
|
|
$container.active(),
|
|
$contained.entityType == EntityType.RECOMMENDATION,
|
|
$contained.type() == $container.type(),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
|
|
end
|
|
|
|
|
|
// Rule unit: X.4
|
|
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
|
|
salience 256
|
|
when
|
|
$equality: Equality(
|
|
$a: a,
|
|
$b: b,
|
|
$a.type() == $b.type(),
|
|
($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
|
|
$a.active(),
|
|
$b.entityType == EntityType.RECOMMENDATION,
|
|
!$b.hasManualChanges()
|
|
)
|
|
then
|
|
$a.addEngines($b.getEngines());
|
|
$b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
|
|
end
|
|
|
|
|
|
// Rule unit: X.5
|
|
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
|
|
salience 256
|
|
when
|
|
$intersection: Intersection(
|
|
$a: a,
|
|
$b: b,
|
|
($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
|
|
$a.active(),
|
|
$b.entityType == EntityType.RECOMMENDATION,
|
|
!$b.hasManualChanges()
|
|
)
|
|
then
|
|
$b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
|
|
end
|
|
|
|
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
|
|
salience 256
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
$container.entityType == EntityType.RECOMMENDATION,
|
|
$container.active(),
|
|
$contained.entityType == EntityType.RECOMMENDATION,
|
|
$container.type() != $contained.type(),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
|
|
end
|
|
|
|
|
|
// Rule unit: X.6
|
|
rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
|
|
salience 32
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
|
|
$container.active(),
|
|
$contained.type() != $container.type(),
|
|
eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT");
|
|
end
|
|
|
|
rule "X.6.1: Remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
|
|
salience 32
|
|
when
|
|
$containment: Containment(
|
|
$container: container,
|
|
$contained: contained,
|
|
($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
|
|
$container.active(),
|
|
$contained.type() != $container.type(),
|
|
eval($container.getTextRange().length() > $contained.getTextRange().length()),
|
|
!$contained.hasManualChanges()
|
|
)
|
|
then
|
|
$contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range");
|
|
end
|
|
|
|
|
|
// Rule unit: X.8
|
|
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
|
|
salience 257
|
|
when
|
|
$equality: Equality(
|
|
$a: a,
|
|
$b: b,
|
|
$a.type() == $b.type(),
|
|
$a.engines contains Engine.IMPORTED,
|
|
$a.active(),
|
|
$b.engines not contains Engine.IMPORTED,
|
|
$a != $b
|
|
)
|
|
then
|
|
$b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
|
|
$a.addEngines($b.getEngines());
|
|
end
|
|
|
|
rule "X.8.1: Remove Entity when intersected by imported Entity"
|
|
salience 256
|
|
when
|
|
$intersection: Intersection(
|
|
$a: a,
|
|
$b: b,
|
|
$a.engines contains Engine.IMPORTED,
|
|
$a.active(),
|
|
$b.engines not contains Engine.IMPORTED,
|
|
$a != $b
|
|
)
|
|
then
|
|
$b.remove("X.8.1", "remove Entity when intersected by imported Entity");
|
|
end
|
|
|
|
|
|
// Rule unit: X.9
|
|
rule "X.9.0: Merge mostly contained signatures"
|
|
when
|
|
$aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
|
|
$signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
|
|
then
|
|
$aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
|
|
$signature.addEngine(LayoutEngine.AI);
|
|
end
|
|
|
|
|
|
// Rule unit: X.10
|
|
rule "X.10.0: remove false positives of ai"
|
|
when
|
|
$anyImage: Image(engines contains LayoutEngine.ALGORITHM)
|
|
$aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
|
|
then
|
|
$aiSignature.remove("X.10.0", "Removed because false positive");
|
|
end
|
|
|
|
|
|
// Rule unit: X.11
|
|
rule "X.11.1: Remove non-manual entity which intersects with a manual entity"
|
|
salience 64
|
|
when
|
|
$intersection: Intersection(
|
|
$a: a,
|
|
$b: b,
|
|
$a.engines contains Engine.MANUAL,
|
|
$a.active(),
|
|
$b.engines not contains Engine.MANUAL
|
|
)
|
|
then
|
|
$b.remove("X.11.1", "remove entity which intersects with a manual entity");
|
|
end
|
|
|
|
rule "X.11.2: Remove non-manual entity which is equal to manual entity"
|
|
salience 70
|
|
when
|
|
$equality: Equality(
|
|
$a: a,
|
|
$b: b,
|
|
$a.type() == $b.type(),
|
|
$a.engines contains Engine.MANUAL,
|
|
$a.active(),
|
|
$b.entityType == EntityType.ENTITY,
|
|
!$b.hasManualChanges(),
|
|
$b.engines not contains Engine.MANUAL
|
|
)
|
|
then
|
|
$a.addEngines($b.getEngines());
|
|
$b.remove("X.11.2", "remove non-manual entity which is equal to manual entity");
|
|
end
|
|
|
|
|
|
//------------------------------------ Dictionary merging rules ------------------------------------
|
|
|
|
// Rule unit: DICT.0
|
|
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
|
|
salience 64
|
|
when
|
|
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
|
|
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
|
|
then
|
|
$entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
|
|
$entity.addEngine(Engine.DOSSIER_DICTIONARY);
|
|
end
|
|
|
|
|
|
//------------------------------------ File attributes rules ------------------------------------
|
|
|
|
// Rule unit: FA.1
|
|
rule "FA.1.0: Remove duplicate FileAttributes"
|
|
salience 64
|
|
when
|
|
$fileAttribute: FileAttribute($label: label, $value: value)
|
|
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
|
|
then
|
|
retract($duplicate);
|
|
end
|
|
|
|
|
|
//------------------------------------ Local dictionary search rules ------------------------------------
|
|
|
|
// Rule unit: LDS.0
|
|
rule "LDS.0.0: Run local dictionary search"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -999
|
|
when
|
|
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
|
|
then
|
|
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
|
|
.forEach(entity -> {
|
|
Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
|
|
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
|
|
});
|
|
end
|