package drools

import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;

import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;

import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;

import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
global RulesLogger logger

//------------------------------------ queries ------------------------------------

query "getFileAttributes"
    $fileAttribute: FileAttribute()
    end

//------------------------------------ Syngenta specific rules ------------------------------------

// Rule unit: SYN.1
rule "SYN.1.0: Recommend CTL/BL laboratory that start with BL or CTL"
    when
        $section: Section(containsString("CT") || containsString("BL"))
    then
        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
        entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
            .forEach(entity -> entity.skip("SYN.1.0", ""));
    end


//------------------------------------ CBI rules ------------------------------------

// Rule unit: CBI.0
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        $section: Section(containsString("et al."))
    then
        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
            .forEach(entity -> {
                entity.skip("CBI.0.0", "Author found by \"et al\" regex");
                dictionary.recommendEverywhere(entity);
            });
    end

rule "CBI.0.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("et al."))
    then
        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
            .forEach(entity -> {
                entity.redact("CBI.0.1", "Author found by \"et al\" regex", "personal_data_geolocation_article_39e3");
                dictionary.recommendEverywhere(entity);
            });
    end

rule "CBI.0.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("et al."))
    then
        entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
            .forEach(entity -> {
                entity.redact("CBI.0.2", "Author found by \"et al\" regex", "vertebrate_study_personal_data_geolocation_article_39e2");
                dictionary.recommendEverywhere(entity);
            });
    end

rule "CBI.0.3: Redact CBI Authors (non vertebrate Study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
    then
        $entity.redact("CBI.0.3", "Author found", "personal_data_geolocation_article_39e3");
    end

rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entity: TextEntity(type() == "CBI_author", dictionaryEntry)
    then
        $entity.redact("CBI.0.4", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2");
    end


// Rule unit: CBI.1
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
    then
        $entity.skip("CBI.1.0", "Address found for Non Vertebrate Study");
    end

rule "CBI.1.1: Redact CBI Address (vertebrate Study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entity: TextEntity(type() == "CBI_address", dictionaryEntry)
    then
        $entity.redact("CBI.1.1", "Address found", "vertebrate_study_personal_data_geolocation_article_39e2");
    end


// Rule unit: CBI.2
rule "CBI.2.0: Do not redact genitive CBI Author"
    when
        $entity: TextEntity(type() == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"))
    then
        entityCreationService.byTextRange($entity.getTextRange(), "CBI_author", EntityType.FALSE_POSITIVE, document)
        .ifPresent(falsePositive -> falsePositive.skip("CBI.2.0", "Genitive Author found"));
    end


// Rule unit: CBI.9
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $table: Table(hasHeader("Author(s)"))
    then
        $table.streamTableCellsWithHeader("Author(s)")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "personal_data_geolocation_article_39e3"));
    end

rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $table: Table(hasHeader("Author"))
    then
        $table.streamTableCellsWithHeader("Author")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "personal_data_geolocation_article_39e3"));
    end


// Rule unit: CBI.10
rule "CBI.10.0: Redact all cells with Header Author(s) as CBI_author (vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $table: Table(hasHeader("Author(s)"))
    then
        $table.streamTableCellsWithHeader("Author(s)")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.0", "Author(s) found", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end

rule "CBI.10.1: Redact all cells with Header Author as CBI_author (vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $table: Table(hasHeader("Author"))
    then
        $table.streamTableCellsWithHeader("Author")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.redact("CBI.10.1", "Author found", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: CBI.11
rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    salience -1
    when
        $table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
    then
        $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
    end


// Rule unit: CBI.16
rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
    when
        $section: Section(!hasTables(),
                          hasEntitiesOfType("published_information"),
                          (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
    then
        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
            .forEach(redactionEntity -> {
                redactionEntity.skipWithReferences(
                        "CBI.16.0",
                        "Published Information found in section",
                        $section.getEntitiesOfType("published_information")
                );
            });
    end

rule "CBI.16.1: Do not redact Names and Addresses if published information found in same table row"
    when
        $table: Table(hasEntitiesOfType("published_information"), hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
        $cellsWithPublishedInformation: TableCell() from $table.streamTableCellsWhichContainType("published_information").toList()
        $tableCell: TableCell(row == $cellsWithPublishedInformation.row) from $table.streamTableCells().toList()
        $authorOrAddress: TextEntity(type() == "CBI_author" || type() == "CBI_address", active()) from $tableCell.getEntities()
    then
        $authorOrAddress.skipWithReferences("CBI.16.1", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $authorOrAddress));
    end


// Rule unit: CBI.17
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
    when
        $section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
    then
        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
            .forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
    end

rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
    when
        $section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
    then
        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
            .forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
    end


// Rule unit: CBI.20
rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
    then
        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
         .forEach(laboratoryEntity -> {
             laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
             dictionary.recommendEverywhere(laboratoryEntity);
         });
    end

rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
    then
         entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
         .forEach(laboratoryEntity -> {
             laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "vertebrate_study_personal_data_geolocation_article_39e2");
             dictionary.recommendEverywhere(laboratoryEntity);
         });
    end


// Rule unit: CBI.23
rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
    then
        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
         .forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "personal_data_geolocation_article_39e3"));
    end

rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
    then
        entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document, 200)
        .forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


//------------------------------------ PII rules ------------------------------------

// Rule unit: PII.0
rule "PII.0.1: Redact all PII (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $pii: TextEntity(type() == "PII", dictionaryEntry)
    then
        $pii.redact("PII.0.1", "Personal Information found", "personal_data_geolocation_article_39e3");
    end

rule "PII.0.2: Redact all PII (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $pii: TextEntity(type() == "PII", dictionaryEntry)
    then
        $pii.redact("PII.0.2", "Personal Information found", "vertebrate_study_personal_data_geolocation_article_39e2");
    end


// Rule unit: PII.1
rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("@"))
    then
        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
            .forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "personal_data_geolocation_article_39e3"));
    end

rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("@"))
    then
        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
            .forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end

rule "PII.1.3: Redact typoed Emails with indicator"
    when
        $section: Section(containsString("@") || containsStringIgnoreCase("mail"))
    then
        entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
            .forEach(emailEntity -> emailEntity.redact("PII.1.3", "Personal information found", "personal_data_geolocation_article_39e3"));
    end


// Rule unit: PII.2
rule "PII.2.1: Redact Phone and Fax by RegEx (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("Contact") ||
                              containsString("Telephone") ||
                              containsString("Phone") ||
                              containsString("Fax") ||
                              containsString("Tel") ||
                              containsString("Ter") ||
                              containsString("Mobile") ||
                              containsString("Fel") ||
                              containsString("Fer"))
    then
        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.1", "Found by Phone and Fax Regex", "personal_data_geolocation_article_39e3"));
    end

rule "PII.2.2: Redact Phone and Fax by RegEx (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("Contact") ||
                              containsString("Telephone") ||
                              containsString("Phone") ||
                              containsString("Fax") ||
                              containsString("Tel") ||
                              containsString("Ter") ||
                              containsString("Mobile") ||
                              containsString("Fel") ||
                              containsString("Fer"))
    then
        entityCreationService.byRegexIgnoreCase("\\b(contact|telephone|phone|ph\\.|fax|tel|ter[^\\w]|mobile|fel[^\\w]|fer[^\\w])[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.2.2", "Found by Phone and Fax Regex", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end

rule "PII.2.3: Redact phone numbers without indicators"
    when
        $section: Section(containsString("+"))
    then
        entityCreationService.byRegex("(\\+[\\dO]{1,2} )(\\([\\dO]{1,3}\\))?[\\d\\-O ]{8,15}", "PII", EntityType.ENTITY, $section)
            .forEach(entity -> entity.redact("PII.2.3", "Personal information found", "personal_data_geolocation_article_39e3"));
    end


// Rule unit: PII.3
rule "PII.3.1: Redact telephone numbers by RegEx (Non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
    then
        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
        .forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "personal_data_geolocation_article_39e3"));
    end

rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(!hasTables(), matchesRegex("[+]\\d{1,}"))
    then
        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
        .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end

rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
    then
        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
        .forEach(entity -> entity.redact("PII.3.4", "Telephone number found by regex", "personal_data_geolocation_article_39e3"));
    end

rule "PII.3.5: Redact telephone numbers by RegEx (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $rowCell: TableCell(matchesRegex("[+]\\d{1,}"))
    then
        entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $rowCell)
        .forEach(entity -> entity.redact("PII.3.5", "Telephone number found by regex", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: PII.5
rule "PII.5.1: Redact line after contact information keywords reduced (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $contactKeyword: String() from List.of("Contact point:",
                                         "Contact:",
                                         "Alternative contact:",
                                         "European contact:")
        $section: Section(containsString($contactKeyword))
    then
        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "personal_data_geolocation_article_39e3"));
    end

rule "PII.5.2: Redact line after contact information keywords reduced (Vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $contactKeyword: String() from List.of("Contact point:",
                                               "Contact:",
                                               "Alternative contact:",
                                               "European contact:")
        $section: Section(containsString($contactKeyword))
    then
        entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
            .forEach(contactEntity -> contactEntity.redact("PII.5.2", "Found after \"" + $contactKeyword + "\" contact keyword", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: PII.6
rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
    then
        Stream.concat(
            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
        )
            .forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "personal_data_geolocation_article_39e3"));
    end

rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
    then
        Stream.concat(
            entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
            entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
        )
            .forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: PII.7
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(getHeadline().containsString("applicant") ||
                          getHeadline().containsString("Primary contact") ||
                          getHeadline().containsString("Alternative contact") ||
                          containsString("Applicant") ||
                          containsString("Telephone number:"))
    then
        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
             Stream.concat(
                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
             ))
            .forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "personal_data_geolocation_article_39e3"));
    end

rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(getHeadline().containsString("applicant") ||
                          getHeadline().containsString("Primary contact") ||
                          getHeadline().containsString("Alternative contact") ||
                          containsString("Applicant") ||
                          containsString("Telephone number:"))
    then
        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
             Stream.concat(
                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
             ))
            .forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: PII.8
rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                          containsStringIgnoreCase("producer of the active substance") ||
                          containsStringIgnoreCase("manufacturer of the active substance") ||
                          containsStringIgnoreCase("manufacturer:") ||
                          containsStringIgnoreCase("Producer or producers of the active substance"))
    then
        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
             Stream.concat(
                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
             ))
            .forEach(entity -> entity.redact("PII.8.1", "Producer was found", "personal_data_geolocation_article_39e3"));
    end

rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsStringIgnoreCase("producer of the plant protection") ||
                          containsStringIgnoreCase("producer of the active substance") ||
                          containsStringIgnoreCase("manufacturer of the active substance") ||
                          containsStringIgnoreCase("manufacturer:") ||
                          containsStringIgnoreCase("Producer or producers of the active substance"))
    then
        Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
                            "Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
             Stream.concat(
                 entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
                 entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
             ))
            .forEach(entity -> entity.redact("PII.8.2", "Producer was found", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
    then
        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
        .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "personal_data_geolocation_article_39e3"));
     end

rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
    then
        entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
        .forEach(entity -> entity.redact("PII.10.1", "Personal information found", "vertebrate_study_personal_data_geolocation_article_39e2"));
     end


// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
    when
        $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
    then
        entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
         .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "personal_data_geolocation_article_39e3"));
    end


// Rule unit: PII.12
rule "PII.12.0: Expand PII entities with salutation prefix"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
    then
        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "personal_data_geolocation_article_39e3"));
    end

rule "PII.12.1: Expand PII entities with salutation prefix"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
    then
        entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
        .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "vertebrate_study_personal_data_geolocation_article_39e2"));
    end


//------------------------------------ Other rules ------------------------------------

// Rule unit: ETC.0
rule "ETC.0.0: Purity Hint"
    when
        $section: Section(containsStringIgnoreCase("purity"))
    then
        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) [<>]{0,1}(100|([1-9]{1}[0-9]{0,1}([.,]{1}[0-9]{1,2})?)) ?% ", "hint_only", EntityType.HINT, 1, $section)
            .forEach(hint -> hint.skip("ETC.0.0", "hint only"));
    end


// Rule unit: ETC.2
rule "ETC.2.1: Redact signatures (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $signature: Image(imageType == ImageType.SIGNATURE)
    then
        $signature.redact("ETC.2.1", "Signature Found", "personal_data_geolocation_article_39e3");
    end

rule "ETC.2.2: Redact signatures (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $signature: Image(imageType == ImageType.SIGNATURE)
    then
        $signature.redact("ETC.2.2", "Signature Found", "vertebrate_study_personal_data_geolocation_article_39e2");
    end


// Rule unit: ETC.3
rule "ETC.3.1: Skip logos (non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $logo: Image(imageType == ImageType.LOGO)
    then
        $logo.skip("ETC.3.1", "Logo Found");
    end

rule "ETC.3.2: Redact logos (vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $logo: Image(imageType == ImageType.LOGO)
    then
        $logo.redact("ETC.3.2", "Logo Found", "vertebrate_study_personal_data_geolocation_article_39e2");
    end


// Rule unit: ETC.5
rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
    when
        FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
        $dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
    end

rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
    salience 256
    when
        not FileAttribute(label == "Confidentiality", value == "confidential")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
        $dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
    end


// Rule unit: ETC.12
rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
    when
        not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
        $dossierRedaction.skip("ETC.12.2", "Dossier dictionary entry found");
    end

rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
    when
        FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
        $dossierRedaction: TextEntity(type() == "dossier_redaction")
    then
        $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
    end


//------------------------------------ AI rules ------------------------------------

// Rule unit: AI.0
rule "AI.0.0: Add all NER Entities of type CBI_author"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
    then
        nerEntities.streamEntitiesOfType("CBI_author")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 40)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
    end


// Rule unit: AI.1
rule "AI.1.0: Combine and add NER Entities as CBI_address"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
    then
        entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
    end


// Rule unit: AI.4
rule "AI.4.0: Add all NER Entities of type Person"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Person"))
    then
        nerEntities.streamEntitiesOfType("Person")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_author", EntityType.RECOMMENDATION, document)
                                                        .ifPresent(e -> e.skip("AI.4.0", "")));
    end


// Rule unit: AI.5
rule "AI.5.0: Combine and add NER Entities as CBI_address"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Location") || hasEntitiesOfType("Address")|| hasEntitiesOfType("Organization"))
    then
        entityCreationService
            .combineNerEntitiesWithConfidence(
                nerEntities,
                "CBI_address",
                EntityType.RECOMMENDATION,
                document,
                Set.of("Organization",
                        "Location",
                        "Address",
                        "ORG",
                        "STREET",
                        "CITY"),
                Set.of("Organization",
                        "Location",
                        "Address",
                        "ORG",
                        "STREET",
                        "POSTAL",
                        "COUNTRY",
                        "CARDINAL",
                        "CITY",
                        "STATE"),
                50,
                3,
                2,
                0.7)
            .forEach(nerEntity -> nerEntity.skip("AI.5.0", ""));
    end


// Rule unit: AI.6
rule "AI.6.0: Add all NER Entities of type Location"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Location"))
    then
        nerEntities.streamEntitiesOfType("Location")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
                                                       .ifPresent(e -> e.skip("AI.6.0", "")));
    end


// Rule unit: AI.7
rule "AI.7.0: Add all NER Entities of type Address"
    salience 999
    when
        nerEntities: NerEntities(hasEntitiesOfType("Address"))
    then
        nerEntities.streamEntitiesOfType("Address")
            .filter(entity -> entity.value().length() > 3)
            .filter(entity -> entity.value().length() < 100)
            .forEach(nerEntity -> entityCreationService.optionalByNerEntityWithConfidence(nerEntity, 0.7, "CBI_address", EntityType.RECOMMENDATION, document)
                                                       .ifPresent(e -> e.skip("AI.7.0", "")));
    end


//------------------------------------ Manual changes rules ------------------------------------

// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $entityToBeResized: TextEntity(matchesAnnotationId($id))
    then
        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
        retract($resizeRedaction);
    end

rule "MAN.0.1: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $imageToBeResized: Image(id == $id)
    then
        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
        retract($resizeRedaction);
    end


// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeRemoved.addManualChange($idRemoval);
        retract($idRemoval);
    end

rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId)
        $imageEntityToBeRemoved: Image($id == id)
    then
        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
        retract($idRemoval);
    end


// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $entityToForce: TextEntity(matchesAnnotationId($id))
    then
        $entityToForce.addManualChange($force);
        retract($force);
    end

rule "MAN.2.1: Apply force redaction to images"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $imageToForce: Image(id == $id)
    then
        $imageToForce.getManualOverwrite().addChange($force);
        retract($force);
    end


// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
    then
        $entityToBeRecategorized.addManualChange($recategorization);
        retract($recategorization);
    end

rule "MAN.3.1: Apply entity recategorization of same type"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
    then
        $entityToBeRecategorized.addManualChange($recategorization);
        retract($recategorization);
    end

rule "MAN.3.2: Apply image recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $imageToBeRecategorized: Image($id == id)
    then
        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
        retract($recategorization);
    end

rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end


// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $imageToBeRecategorized: Image($id == id)
    then
        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
        retract($legalBasisChange)
    end

rule "MAN.4.1: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeChanged.addManualChange($legalBasisChange);
        retract($legalBasisChange)
    end


//------------------------------------ Entity merging rules ------------------------------------

// Rule unit: X.0
rule "X.0.0: Remove Container Entity Contained by Container of Same Type"
    salience 65
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.type() == $contained.type(),
            $container.entityType == $contained.entityType,
            $container != $contained,
            !$container.removed(),
            !$container.hasManualChanges(),
            !$contained.hasManualChanges(),
            !$contained.removed()
        )
        not TextEntity(
            getTextRange().equals($container.getTextRange()),
            type() == $container.type(),
            entityType == EntityType.DICTIONARY_REMOVAL,
            engines contains Engine.DOSSIER_DICTIONARY,
            !hasManualChanges()
        )
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
end

rule "X.0.1: Remove Container and Contained Entities with Manual Changes"
    salience 65
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.type() == $contained.type(),
            $container.entityType == $contained.entityType,
            $container != $contained,
            !$container.removed(),
            $container.hasManualChanges(),
            !$contained.hasManualChanges(),
            !$contained.removed()
        )
    then
        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
end


// Rule unit: X.2
rule "X.2.0: Remove ENTITY and FALSE_POSITIVE Containing It"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_POSITIVE,
            $container.active(),
            $contained.entityType == EntityType.ENTITY,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
end

rule "X.2.1: Remove HINT Contained by FALSE_POSITIVE"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_POSITIVE,
            $container.active(),
            $contained.entityType == EntityType.HINT,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE");
end


rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_RECOMMENDATION,
            $container.active(),
            $contained.entityType == EntityType.RECOMMENDATION,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
end


// Rule unit: X.4
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
    salience 256
    when
        $equality: Equality(
            $a: a,
            $b: b,
            $a.type() == $b.type(),
            ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
            $a.active(),
            $b.entityType == EntityType.RECOMMENDATION,
            !$b.hasManualChanges()
        )
    then
        $a.addEngines($b.getEngines());
        $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
    end


// Rule unit: X.5
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
            $a.active(),
            $b.entityType == EntityType.RECOMMENDATION,
            !$b.hasManualChanges()
        )
    then
        $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
    end

rule "X.5.1: Remove Lower Priority RECOMMENDATION Contained by Higher Priority RECOMMENDATION"
    salience 256
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.RECOMMENDATION,
            $container.active(),
            $contained.entityType == EntityType.RECOMMENDATION,
            $container.type() != $contained.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
end


// Rule unit: X.6
rule "X.6.0: Remove Lower Rank Entity Contained by ENTITY or HINT"
    salience 32
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
            $container.active(),
            $contained.type() != $container.type(),
            eval(dictionary.getDictionaryRank($contained.type()) < dictionary.getDictionaryRank($container.type())),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.6.0", "remove Entity of lower rank when contained by entity of type ENTITY or HINT");
    end

rule "X.6.1: Remove Smaller Text Range Contained by ENTITY or HINT"
    salience 32
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            ($container.entityType == EntityType.ENTITY || $container.entityType == EntityType.HINT),
            $container.active(),
            $contained.type() != $container.type(),
            eval($container.getTextRange().length() > $contained.getTextRange().length()),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.6.1", "remove Entity when contained in another entity of type ENTITY or HINT with larger text range");
end


// Rule unit: X.8
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
    salience 257
    when
        $equality: Equality(
            $a: a,
            $b: b,
            $a.type() == $b.type(),
            $a.engines contains Engine.IMPORTED,
            $a.active(),
            $b.engines not contains Engine.IMPORTED,
            $a != $b
        )
    then
        $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
        $a.addEngines($b.getEngines());
    end

rule "X.8.1: Remove Entity when intersected by imported Entity"
    salience 256
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            $a.engines contains Engine.IMPORTED,
            $a.active(),
            $b.engines not contains Engine.IMPORTED,
            $a != $b
        )
    then
        $b.remove("X.8.1", "remove Entity when intersected by imported Entity");
    end


// Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures"
    when
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
    then
        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
        $signature.addEngine(LayoutEngine.AI);
    end


// Rule unit: X.10
rule "X.10.0: remove false positives of ai"
    when
        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
    then
        $aiSignature.remove("X.10.0", "Removed because false positive");
    end


// Rule unit: X.11
rule "X.11.1: Remove non-manual entity which intersects with a manual entity"
    salience 64
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            $a.engines contains Engine.MANUAL,
            $a.active(),
            $b.engines not contains Engine.MANUAL
        )
    then
        $b.remove("X.11.1", "remove entity which intersects with a manual entity");
    end

rule "X.11.2: Remove non-manual entity which is equal to manual entity"
    salience 70
    when
        $equality: Equality(
            $a: a,
            $b: b,
            $a.type() == $b.type(),
            $a.engines contains Engine.MANUAL,
            $a.active(),
            $b.entityType == EntityType.ENTITY,
            !$b.hasManualChanges(),
            $b.engines not contains Engine.MANUAL
        )
    then
        $a.addEngines($b.getEngines());
        $b.remove("X.11.2", "remove non-manual entity which is equal to manual entity");
    end


//------------------------------------ Dictionary merging rules ------------------------------------

// Rule unit: DICT.0
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
    salience 64
    when
        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
    end


//------------------------------------ File attributes rules ------------------------------------

// Rule unit: FA.1
rule "FA.1.0: Remove duplicate FileAttributes"
    salience 64
    when
        $fileAttribute: FileAttribute($label: label, $value: value)
        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
    then
        retract($duplicate);
    end


//------------------------------------ Local dictionary search rules ------------------------------------

// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    salience -999
    when
        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
    then
        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
            .forEach(entity ->  {
                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
            });
    end