package drools

import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;

import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;

import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;

import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
global RulesLogger logger

//------------------------------------ queries ------------------------------------

query "getFileAttributes"
    $fileAttribute: FileAttribute()
    end

//------------------------------------ Table extraction rules ------------------------------------

// Rule unit: TAB.0
rule "TAB.0.0: Study Type File Attribute"
    when
        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
            && containsAnyString("OECD", "EPA", "OPPTS"))
    then
        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
            .forEach(fileAttribute -> insert(fileAttribute));
        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
            .forEach(fileAttribute -> insert(fileAttribute));
    end

rule "TAB.0.1: Guidelines"
    when
        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
    then
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
    end


// Rule unit: TAB.1
rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Full Table"))
        $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
        $tableCell: TableCell(!header) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
    end


// Rule unit: TAB.2
rule "TAB.2.0: Individual row extraction (Clinical Signs)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Individual Rows Extraction"))
        $table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
        TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
        $tableCell: TableCell($row == row) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
    end


// Rule unit: TAB.3
rule "TAB.3.0: Individual column extraction (Strain)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Individual Column"))
        $table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
    then
        $table.streamTableCellsWithHeader("Sex")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
    end


// Rule unit: TAB.4
rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Combined Columns"))
        $table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
    then
        $table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
    end

rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Combined Columns"))
        $table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
    then
        $table.streamTableCellsWithHeader("Mortality")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
    end


// Rule unit: TAB.5
rule "TAB.5.0: Targeted cell extraction"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Value Extraction"))
        $table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
        TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
        $dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
    then
        entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
    end


// Rule unit: TAB.6
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
    when
        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
    then
        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
    end


// Rule unit: TAB.7
rule "TAB.7.0: Indicator (Species)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Entity-Based"))
        $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
        TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
        TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
        $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
    end


//------------------------------------ AI rules ------------------------------------

// Rule unit: AI.4
rule "AI.4.0: Add all NER Entities of type Person"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Person"))
    then
        nerEntities.streamEntitiesOfType("Person")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_author", EntityType.RECOMMENDATION, document)
                                                        .ifPresent(e -> e.skip("AI.4.0", "")));
    end


// Rule unit: AI.5
rule "AI.5.0: Combine and add NER Entities as CBI_address"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Location") || hasEntitiesOfType("Address")|| hasEntitiesOfType("Organization"))
    then
        entityCreationService
            .combineNerEntitiesWithConfidence(
                nerEntities,
                "CBI_address",
                EntityType.RECOMMENDATION,
                document,
                Set.of("Organization",
                        "Location",
                        "Address",
                        "ORG",
                        "STREET",
                        "CITY"),
                Set.of("Organization",
                        "Location",
                        "Address",
                        "ORG",
                        "STREET",
                        "POSTAL",
                        "COUNTRY",
                        "CARDINAL",
                        "CITY",
                        "STATE"),
                50,
                3,
                2,
                0.7)
            .forEach(nerEntity -> nerEntity.skip("AI.5.0", ""));
    end


// Rule unit: AI.6
rule "AI.6.0: Add all NER Entities of type Location"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Location"))
    then
        nerEntities.streamEntitiesOfType("Location")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
                                                       .ifPresent(e -> e.skip("AI.6.0", "")));
    end


// Rule unit: AI.7
rule "AI.7.0: Add all NER Entities of type Address"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Address"))
    then
        nerEntities.streamEntitiesOfType("Address")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
                                                       .ifPresent(e -> e.skip("AI.7.0", "")));
    end


//------------------------------------ Manual changes rules ------------------------------------

// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $entityToBeResized: TextEntity(matchesAnnotationId($id))
    then
        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
        retract($resizeRedaction);
        update($entityToBeResized);
        $entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
    end

rule "MAN.0.1: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $imageToBeResized: Image(id == $id)
    then
        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
        retract($resizeRedaction);
        update($imageToBeResized);
        update($imageToBeResized.getParent());
    end


// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
        update($entityToBeRemoved);
        retract($idRemoval);
        $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
    end

rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId)
        $imageEntityToBeRemoved: Image($id == id)
    then
        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
        update($imageEntityToBeRemoved);
        retract($idRemoval);
        update($imageEntityToBeRemoved.getParent());
    end


// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $entityToForce: TextEntity(matchesAnnotationId($id))
    then
        $entityToForce.getManualOverwrite().addChange($force);
        update($entityToForce);
        $entityToForce.getIntersectingNodes().forEach(node -> update(node));
        retract($force);
    end

rule "MAN.2.1: Apply force redaction to images"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $imageToForce: Image(id == $id)
    then
        $imageToForce.getManualOverwrite().addChange($force);
        update($imageToForce);
        update($imageToForce.getParent());
        retract($force);
    end


// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
    then
        $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
        update($entityToBeRecategorized);
        retract($recategorization);
    end

rule "MAN.3.1: Apply entity recategorization of same type"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
    then
        $entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
        retract($recategorization);
    end

rule "MAN.3.2: Apply image recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $imageToBeRecategorized: Image($id == id)
    then
        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
        update($imageToBeRecategorized);
        update($imageToBeRecategorized.getParent());
        retract($recategorization);
    end

rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end


// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $imageToBeRecategorized: Image($id == id)
    then
        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
        update($imageToBeRecategorized)
        retract($legalBasisChange)
    end

rule "MAN.4.1: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
        update($entityToBeChanged)
        retract($legalBasisChange)
    end


//------------------------------------ Entity merging rules ------------------------------------

// Rule unit: X.0
rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), !hasManualChanges())
        not TextEntity(getTextRange().equals($larger.getTextRange()), type() == $type, entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, !hasManualChanges())
        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
        retract($contained);
    end

rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
    salience 65
    when
        $larger: TextEntity($type: type(), $entityType: entityType, !removed(), hasManualChanges())
        $contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
    then
        $contained.getIntersectingNodes().forEach(node -> update(node));
        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
        retract($contained);
    end


// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
    then
        $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
        retract($entity)
    end

rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
    salience 64
    when
        $falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
        $entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
        retract($entity)
    end


// Rule unit: X.3
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
    salience 64
    when
        $falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
        $recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
        retract($recommendation);
    end


// Rule unit: X.4
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
    salience 256
    when
        $entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
        $recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $entity.addEngines($recommendation.getEngines());
        $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
        retract($recommendation);
    end


// Rule unit: X.5
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
        $recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
        retract($recommendation);
    end

rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
    salience 256
    when
        $entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
        $recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
    then
        $recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
        retract($recommendation);
    end


// Rule unit: X.6
rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
    salience 32
    when
        $higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
        $lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
    then
        $lowerRank.getIntersectingNodes().forEach(node -> update(node));
        $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
        retract($lowerRank);
    end

rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
    salience 32
    when
         $outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
         $inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
    then
        $inner.getIntersectingNodes().forEach(node -> update(node));
        $inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
        retract($inner);
    end


// Rule unit: X.7
rule "X.7.0: Remove all images"
    salience 512
    when
        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
    then
        $image.remove("X.7.0", "remove all images");
        retract($image);
    end


// Rule unit: X.8
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
    salience 257
    when
        $entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
        $other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
    then
        $other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
        $entity.addEngines($other.getEngines());
        retract($other);
    end

rule "X.8.1: Remove Entity when intersected by imported Entity"
    salience 256
    when
        $entity: TextEntity(engines contains Engine.IMPORTED, active())
        $other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
    then
        $other.remove("X.8.1", "remove Entity when intersected by imported Entity");
        retract($other);
    end


// Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures"
    when
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
    then
        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
        $signature.addEngine(LayoutEngine.AI);
    end


// Rule unit: X.10
rule "X.10.0: remove false positives of ai"
    when
        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
    then
        $aiSignature.remove("X.10.0", "Removed because false positive");
    end


// Rule unit: X.11
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
    salience 64
    when
        $manualEntity: TextEntity(engines contains Engine.MANUAL, active())
        $nonManualEntity: TextEntity(intersects($manualEntity), engines not contains Engine.MANUAL)
    then
        $nonManualEntity.remove("X.11.1", "remove entity which intersects with a manual entity");
        retract($nonManualEntity);
    end

rule "X.11.2: Remove non manual entity which are equal to manual entity"
    salience 70
    when
        $manualEntity: TextEntity(engines contains Engine.MANUAL, active(), $type: type())
        $nonManualEntity: TextEntity(getTextRange().equals($manualEntity.getTextRange()), type() == $type, entityType == EntityType.ENTITY, !hasManualChanges(), engines not contains Engine.MANUAL)
    then
        $manualEntity.addEngines($nonManualEntity.getEngines());
        $nonManualEntity.remove("X.11.2", "remove non manual entity which are equal to manual entity");
        retract($nonManualEntity);
    end


//------------------------------------ Dictionary merging rules ------------------------------------

// Rule unit: DICT.0
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
    salience 64
    when
        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.getIntersectingNodes().forEach(node -> update(node));
        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
    end


//------------------------------------ File attributes rules ------------------------------------

// Rule unit: FA.1
rule "FA.1.0: Remove duplicate FileAttributes"
    salience 64
    when
        $fileAttribute: FileAttribute($label: label, $value: value)
        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
    then
        retract($duplicate);
    end


//------------------------------------ Local dictionary search rules ------------------------------------

// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    salience -999
    when
        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
    then
        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
            .forEach(entity ->  {
                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
            });
    end