package drools

import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;

import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;

import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.LayoutEngine;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;

import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;

global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
global RulesLogger logger


//------------------------------------ queries ------------------------------------

query "getFileAttributes"
    $fileAttribute: FileAttribute()
    end

//------------------------------------ H rules ------------------------------------

// Rule unit: H.0
rule "H.0.0: retract table of contents page"
    when
        $page: Page(getMainBodyTextBlock().getSearchText().contains("........") || (getMainBodyTextBlock().getSearchText().contains("APPENDICES") && getMainBodyTextBlock().getSearchText().contains("TABLES")))
        $node: SemanticNode(onPage($page.getNumber()), !onPage($page.getNumber() -1), getType() != NodeType.IMAGE)
    then
        retract($node);
    end


// Rule unit: H.1
rule "H.1.0: Ignore Table of Contents"
    salience 10
    when
        $tocHeadline: Headline(containsString("CONTENTS"))
        $page: Page() from $tocHeadline.getParent().getPages()
        $node: SemanticNode(this != $tocHeadline, getType() != NodeType.IMAGE, onPage($page.getNumber()), !onPage($page.getNumber() -1))
    then
        retract($node);
    end


// Rule unit: H.2
rule "H.2.0: Show headlines"
    when
        $headline: Headline()
    then
        entityCreationService.bySemanticNode($headline, "headline", EntityType.HINT);
    end


// Rule unit: H.3
rule "H.3.0: Study Type File Attribute"
    when
        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
        $section: Section(
            (containsString("DATA REQUIREMENT") || containsString("TEST GUIDELINE") || containsString("MÉTODO(S) DE REFERÊNCIA(S):"))
            ,(containsString("OECD") || containsString("EPA") || containsString("OPPTS"))
        )
    then
      Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $section.getTextBlock()),
                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()),
                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $section.getTextBlock())).flatMap(Collection::stream).findFirst()
             .map(textRange -> $section.getTextBlock().subSequence(textRange).toString())
             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
             .ifPresent(fileAttribute -> insert(fileAttribute));
    end

rule "H.3.1: Study Type File Attribute in Headlines"
    when
        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
        $page: Page($pageNumber:number,
            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
    then
      Stream.of(RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1, $headline.getTextBlock()),
                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $headline.getTextBlock()),
                RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD) Guideline (4\\d{2})", 1, $headline.getTextBlock())).flatMap(Collection::stream).findFirst()
             .map(textRange -> $headline.getTextBlock().subSequence(textRange).toString())
             .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
             .ifPresent(fileAttribute -> insert(fileAttribute));
    end

//------------------------------------ General documine rules ------------------------------------

// Rule unit: DOC.1
rule "DOC.1.0: Guidelines"
     when
         $section: Section(
            (
                containsString("DATA REQUIREMENT")
                || containsString("TEST GUIDELINE")
                || containsString("MÉTODO(S) DE REFERÊNCIA(S):")
            )
            && (
                containsString("OECD")
                || containsString("EPA")
                || containsString("OPPTS")
            )
         )
     then
        entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline found", "n-a")
        );
        entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline found", "n-a")
        );
        entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "EPA Guideline found", "n-a")
        );
        entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "EC Guideline found", "n-a")
        );
        entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "EC Guideline found", "n-a")
        );
        entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "EPA Guideline found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline no. found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity ->
             entity.apply("DOC.1.0", "OECD Guideline number found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline number found", "n-a")
        );
        entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $section).forEach(entity ->
            entity.apply("DOC.1.0", "OECD Guideline found", "n-a")
        );
     end

rule "DOC.1.2: Guidelines"
     when
         $section: Section(
            (
                containsString("DATA REQUIREMENT")
                || containsString("TEST GUIDELINE")
                || containsString("MÉTODO(S) DE REFERÊNCIA(S):")
            )
            && (
                containsString("OECD")
                || containsString("EPA")
                || containsString("OPPTS")
            )
            && (
                hasEntitiesOfType("oecd_guideline")
                || hasEntitiesOfType("epa_guideline")
                || hasEntitiesOfType("ec_guideline")
            )
         )
     then
         $section.getEntitiesOfType(List.of("oecd_guideline","ec_guideline", "epa_guideline")).forEach(entity -> {
            entity.apply("DOC.1.2", "OECD guideline found.", "n-a");
         });
     end

rule "DOC.1.3: Guidelines"
     when
         $section: Section(
            (
                hasEntitiesOfType("oecd_guideline")
                || hasEntitiesOfType("epa_guideline")
                || hasEntitiesOfType("ec_guideline")
            )
            && !(
                (
                    containsString("DATA REQUIREMENT")
                    || containsString("TEST GUIDELINE")
                    || containsString("MÉTODO(S) DE REFERÊNCIA(S):")
                )
                && (
                    containsString("OECD")
                    || containsString("EPA")
                    || containsString("OPPTS")
                )
            )
         )
     then
         $section.getEntitiesOfType(List.of("oecd_guideline", "ec_guideline", "epa_guideline")).forEach(entity -> {
             entity.remove("DOC.1.3", "removed");
             retract(entity);
         });
     end

rule "DOC.1.4: Guideline in Headlines"
    when
        $page: Page($pageNumber:number,
            getMainBodyTextBlock().getSearchText().contains("DATA REQUIREMENT")
            || getMainBodyTextBlock().getSearchText().contains("TEST GUIDELINE")
            || getMainBodyTextBlock().getSearchText().contains("MÉTODO(S) DE REFERÊNCIA(S):"))
        $headline: Headline(onPage($pageNumber), containsString("OECD") || containsString("EPA")|| containsString("OPPTS"))
    then
        entityCreationService.byRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
        );
        entityCreationService.byRegex("OECD[\\s,]{1}(?:.{1,40}.(?>Procedure|Method).{1,20}\\d{3,4}(?>.{1,100}\\d{4}\\))?|\\[.{1,20}.Skin.{1,20}\\]|[\\d\\s,\\(\\)]{7,10}|[\\w\\.\\s]{1,15}[\\d]{3}\\s\\(\\d{4}\\)|.{0,20}[N|n]umber\\s\\d{3}.{0,1}|Test Guideline \\d{3})", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
        );
        entityCreationService.byRegex("EPA (OPPTS )?\\d{3}[. ]\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
        );
        entityCreationService.byRegex("EC (Directive )?(No\\.? )?\\d{3,4}\\/\\d{3,4}((,? B(\\.| )\\d{1,2}\\.?)? \\(\\d{4}\\))?", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
        );
        entityCreationService.byRegex("Commission Regulation \\(EC\\) No \\d{3}\\/\\d{4}", "ec_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "EC Guideline found", "n-a")
        );
        entityCreationService.byRegex("OECD Method 4\\d{2}.{5,40}\\(.{5,40}\\d{4}\\)", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("OPPTS (Guideline Number )?\\d{3}\\.\\d{4}( \\(\\d{4}\\))?", "epa_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "EPA Guideline found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline no. found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY,1, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
             entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY,2, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline year found", "n-a")
        );
        entityCreationService.byRegex("(?<=OECD) Guideline (4\\d{2})", "oecd_guideline_number", EntityType.ENTITY,1, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline number found", "n-a")
        );
        entityCreationService.byRegex("OECD Guideline 4\\d{2}", "oecd_guideline", EntityType.ENTITY, $headline).forEach(entity ->
            entity.apply("DOC.1.4", "OECD Guideline found", "n-a")
        );
    end

// Rule unit: DOC.2
rule "DOC.2.0: Report number"
    when
       $section: Section(containsString("LABORATORY PROJECT ID") , containsString("Report Number:"))
    then
        entityCreationService.lineAfterString("Report Number:", "report_number", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {
            entity.apply("DOC.2.0", "Report number found", "n-a");
        });
    end


// Rule unit: DOC.3
rule "DOC.3.0: Experimental Starting Date"
    when
        $section: Section(containsString("Experimental I. Starting Date:") || containsString("Experimental II. Starting Date:") || containsStringIgnoreCase("experimental start date") || containsStringIgnoreCase("experimental starting date"))
    then
        entityCreationService.lineAfterStrings(
             List.of("Experimental start date",
                     "Experimental start date:",
                     "Experimental Starting Date",
                     "Experimental Starting Date:",
                     "Experimental starting date",
                     "Experimental starting date:",
                     "Experimental Start Date",
                     "Experimental Start Date:",
                     "Experimental I. Starting Date:",
                     "Experimental II. Starting Date:"), "experimental_start_date", EntityType.ENTITY, $section).forEach(entity -> {
                        entity.apply("DOC.3.0", "Experimental start date found", "n-a");
                     });
    end


// Rule unit: DOC.4
rule "DOC.4.0: Experimental Completion Date"
    when
        $section: Section(containsStringIgnoreCase("experimental termination date") || containsStringIgnoreCase("experimental completion date"))
    then
        entityCreationService.lineAfterStrings(
                 List.of("Experimental termination date",
                 "Experimental termination date:",
                 "Experimental Completion Date",
                 "Experimental Completion Date:",
                 "Experimental completion date",
                 "Experimental completion date:",
                 "Experimental Termination Date",
                 "Experimental Termination Date:"), "experimental_end_date", EntityType.ENTITY, $section).forEach(entity -> {
                     entity.apply("DOC.4.0", "Experimental end date found", "n-a");
                 });
    end


// Rule unit: DOC.5
rule "DOC.5.0: Ignore species and strain in irrelevant study types"
     salience 1
     when
         FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","471","474","487"))
         $section: Section(hasEntitiesOfType("species") || hasEntitiesOfType("strain"))
     then
         $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
             entity.remove("DOC.5.0", "removed");
             retract(entity);
         });
     end

rule "DOC.5.1: Hide all skipped species and strains except in the relevant sections"
     salience 1
     when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
        $section: Section(
            (hasEntitiesOfType("species") || hasEntitiesOfType("strain"))
            && !(
                anyHeadlineContainsStringIgnoreCase("test system")
                || anyHeadlineContainsStringIgnoreCase("animals")
                || anyHeadlineContainsStringIgnoreCase("specification")
            )
        )
     then
         $section.getEntitiesOfType(List.of("species", "strain")).forEach(entity -> {
             entity.remove("DOC.5.1", "removed");
             retract(entity);
         });
     end

rule "DOC.5.2: Species"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
        $section: Section(hasEntitiesOfType("species"))
    then
        $section.getEntitiesOfType("species").forEach(entity -> {
            entity.apply("DOC.5.2", "Species found.", "n-a");
            entity.setValue(entity.getValue().toLowerCase());
        });
    end

rule "DOC.5.3: Strain"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436"))
        $section: Section(
            hasEntitiesOfType("species")
            && hasEntitiesOfType("strain")
            && (
                anyHeadlineContainsStringIgnoreCase("test system")
                || anyHeadlineContainsStringIgnoreCase("animals")
                || anyHeadlineContainsStringIgnoreCase("specification")
            )
        )
    then
        $section.getEntitiesOfType("strain").forEach(entity -> {
            entity.apply("DOC.5.3", "Strain found.", "n-a");
        });
    end


// Rule unit: DOC.6
rule "DOC.6.0: study title by document structure"
    when
       $table: Table(onPage(1),
                    (containsString("Final Report") || containsString("SPL")),
                     numberOfRows == 1,
                     numberOfCols == 1)
       $tableCell: TableCell(row == 1, col == 1) from $table.streamTableCells().toList()
       $paragraph: Paragraph(previousSibling.isPresent(), nextSibling.isPresent()) from $tableCell.streamChildren().toList()
    then
        entityCreationService.bySemanticNode($paragraph, "title", EntityType.ENTITY).ifPresent(entity -> {
            entity.apply("DOC.6.0", "Study title found", "n-a");
        });
     end

rule "DOC.6.1: study title"
    when
        $table: Table(onPage(1), (containsString("Final Report") || containsString("SPL")))
    then
        entityCreationService.byRegexWithLineBreaksIgnoreCase("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $table).findFirst().ifPresent(entity -> {
            entity.apply("DOC.6.1", "Title found", "n-a");
        });
     end

rule "DOC.6.2: study title"
    when
        not Table(onPage(1), (containsString("Final Report") || containsString("SPL")))
        $section: Section(onPage(1), (containsString("Final Report") || containsString("SPL")))
    then
        entityCreationService.byRegexWithLineBreaksIgnoreCase("(?<=\\n)[\\w\\W]{1,300}(?=\\nFinal Report)", "title", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {
            entity.apply("DOC.6.2", "Title found", "n-a");
        });
     end


// Rule unit: DOC.7
rule "DOC.7.0: Performing Laboratory (Name)"
    when
       $section: Section(containsString("PERFORMING LABORATORY:"))
    then
        entityCreationService.lineAfterString("PERFORMING LABORATORY:", "laboratory_name", EntityType.ENTITY, $section).findFirst().ifPresent(entity -> {
            entity.apply("DOC.7.0", "Performing Laboratory found", "n-a");
        });
    end

rule "DOC.7.1: Performing Laboratory (Country)"
     when
         nerEntities: NerEntities(hasEntitiesOfType("COUNTRY"))
         $section: Section(containsString("PERFORMING LABORATORY:"))
     then
         nerEntities.streamEntitiesOfType("COUNTRY")
                     .filter(nerEntity -> $section.getTextRange().contains(nerEntity.textRange()))
                     .map(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, "laboratory_country", EntityType.ENTITY, $section))
                     .flatMap(Optional::stream)
                     .collect(Collectors.toList())
                     .forEach(entity -> {
                          entity.apply("DOC.7.1", "Performing Laboratory found", "n-a");
                     });
     end

rule "DOC.7.2: Performing Laboratory (Country & Name) from dict"
    when
        $section: Section(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
        $countryOrNameFromDictionary: TextEntity(type() == "laboratory_country" || type() == "laboratory_name", $type: type, isDictionaryEntry()) from $section.getEntities()
    then
        $countryOrNameFromDictionary.apply("DOC.7.2", "Performing " + $type + " dictionary entry found.");
    end

rule "DOC.7.3: Performing Laboratory (Country) from dict"
    when
        $section: Section(
            (hasEntitiesOfType("laboratory_country") || hasEntitiesOfType("laboratory_name"))
            && !(containsString("PERFORMING LABORATORY:") || (containsString("PERFORMING") && containsString("LABORATORY:")))
        )
    then
        $section.getEntitiesOfType(List.of("laboratory_country", "laboratory_name")).forEach(entity -> {
            entity.remove("DOC.7.3", "removed");
            retract(entity);
        });
    end


// Rule unit: DOC.8
rule "DOC.8.0: GLP Study"
    when
        $headline: Headline(containsString("GOOD LABORATORY PRACTICE COMPLIANCE")
                                        || containsString("GOOD LABORATORY PRACTICE COMPLIANCE STATEMENT")
                                        || (containsString("DECLARACAO DE CONFORMIDADE") && containsString("PRATICAS DE LABORATORIO"))
                                        || containsString("GLP Certificate")
                                        || containsString("GLP Certificates")
                                        || containsString("GOOD LABORATORY PRACTICE (GLP) CERTIFICATE")
                                        || containsString("Good Laboratory Practice Certificate")
                                        || containsString("STATEMENT OF GLP COMPLIANCE AND AUTHENTICATION"))
    then
        entityCreationService.bySemanticNode($headline, "glp_study", EntityType.ENTITY).ifPresent(entity -> {
            entity.apply("DOC.8.0", "GLP Study found", "n-a");
        });
    end


// Rule unit: DOC.9
rule "DOC.9.0: Batch number from CoA"
    when
        $section: Section(
            (
                anyHeadlineContainsString("Analytical Report")
                || anyHeadlineContainsStringIgnoreCase("Certificate of Analysis")
                || containsStringIgnoreCase("Certificate of Analysis")
            )
            && (
                containsStringIgnoreCase("batch")
                || containsStringIgnoreCase("bath")
                || containsStringIgnoreCase("barch")
                || containsStringIgnoreCase("bateb")
            )
            && (
                containsStringIgnoreCase("identification")
                || containsStringIgnoreCase("ldentitfication")
                || containsStringIgnoreCase("wentification")
                || containsStringIgnoreCase("mentification")
                || containsStringIgnoreCase("kientification")
                || containsStringIgnoreCase("reference number")
                || containsStringIgnoreCase("test substance")
            )
        )
    then
        entityCreationService.lineAfterStrings(List.of("Batch Identification",
            "(Batch Identification):",
            "Bateb Identification",
            "Batch Wentification",
            "Batch Mentification",
            "Batch Kientification",
            "Barch Identification",
            "Bath ldentitfication",
            "Batch of test substance :"), "batch_number", EntityType.ENTITY, $section).forEach(entity -> {
                entity.apply("DOC.9.0", "Batch number found in CoA", "n-a");
        });
   end

rule "DOC.9.1: Batch number"
    when
        $section: Section(
            (
                anyHeadlineContainsStringIgnoreCase("Test Substance")
                || anyHeadlineContainsStringIgnoreCase("Test and Control Substances")
                || anyHeadlineContainsStringIgnoreCase("Test Item")
            )
            && !(
                anyHeadlineContainsString("component")
                || anyHeadlineContainsString("reference")
                || anyHeadlineContainsString("blank")
            )
            && containsStringIgnoreCase("batch")
        )
    then
        Stream.of(entityCreationService.byRegex("Batch ID ([A-Z\\d\\-]{7,14})", "batch_number", EntityType.ENTITY, 1, $section),
                    entityCreationService.lineAfterStrings(List.of("Batch Identification",
                      "Batch number:",
                      "Batch reference number:",
                      "Batch:",
                      "Batch/Lot number:",
                      "Batch (Lot) Number:",
                      "Batch Number:",
                      "Batch Nº:",
                      "Batch no:"
                      ), "batch_number", EntityType.ENTITY, $section)).flatMap(a -> a)
                      .forEach(entity -> {
                          entity.apply("DOC.9.1", "Batch number found", "n-a");
                      });
   end

rule "DOC.9.2: Batch number"
    when
        $section: Section(
            (
               anyHeadlineContainsStringIgnoreCase("Test Substance")
                || anyHeadlineContainsStringIgnoreCase("Test and Control Substances")
                || anyHeadlineContainsStringIgnoreCase("Test Item")
            )
            && !(
                anyHeadlineContainsString("component")
                || anyHeadlineContainsString("reference")
                || anyHeadlineContainsString("blank")
            )
            && containsStringIgnoreCase("batch")
        )
        $batchNumber: String() from List.of("Batch Identification",
                                            "Batch number:",
                                            "Batch reference number:",
                                            "Batch:",
                                            "Batch/Lot number:",
                                            "Batch (Lot) Number:",
                                            "Batch Number:",
                                            "Batch Nº:",
                                            "Batch no:")
        $table: Table(containsStringIgnoreCase($batchNumber)) from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
    then
        entityCreationService.lineAfterStringAcrossColumnsIgnoreCase($batchNumber, "batch_number", EntityType.ENTITY, $table).forEach(entity -> {
            entity.apply("DOC.9.2", "Batch number found", "n-a");
         });
   end


// Rule unit: DOC.10
rule "DOC.10.0: Conclusions - LD50, LC50, Confidence"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","425","436"))
        $section: Section(
               (getHeadline().containsStringIgnoreCase("Conclusion") || anyHeadlineContainsStringIgnoreCase("Lethality"))
            && (containsString("LD") || containsString("LC") || containsString("50") || containsString("LD50") || containsString("lethal concentration") || containsString("lethal dose"))
            && (
                containsString("greater than")
                || containsString("higher than")
                || containsString("above")
                || containsString("in excess")
                || containsString("exceeds")
                || containsString("was found to be")
                || containsString("was calculated to be")
                || containsString("estimated to be")
            )
        )
    then
        entityCreationService.byRegexIgnoreCase("(L[D|C]\\s?50|lethal concentration|lethal dose).{1,200}(greater than|considered to be above|in excess of|exceeds|higher than)", "ld50_greater", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.10.0", "LD50 greater than found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("\\b(?:(?:greater|higher) than|considered to be above|(?:was|is) (?:found|estimated) to be|was calculated to be|in excess of|exceeds|equal to)\\s?([\\d\\.]{1,6})\\s?mg\\/(?:kg|L)", "ld50_value", EntityType.ENTITY,1, $section).forEach(entity -> {
            entity.apply("DOC.10.0", "LD50 value found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_minimal", EntityType.ENTITY,1, $section).forEach(entity -> {
            entity.apply("DOC.10.0", "Minimal Confidence found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("confidence interval (?:is )?([\\d\\.]{2,6}).{0,20} to (?:greater than )?([\\d\\.]{2,6})", "confidence_maximal", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.10.0", "Maximal Confidence found", "n-a");
        });
    end


// Rule unit: DOC.11
rule "DOC.11.0: Guideline Deviation"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
        $section: Section(
            (getHeadline().containsStringIgnoreCase("General Information") || containsString("GENERAL INFORMATION"))
            && (containsStringIgnoreCase("from the") || containsStringIgnoreCase("to the"))
        )
    then
        entityCreationService.betweenRegexes("(?:Deviations? from the [G|g]uidelines?)(?: and| or)?( the)?(?: Study Plan)?", "(?:(?:Deviations? from the Study Plan)|(?:Performing laboratory test)|(?:Other)|(?:Retention of [S|s]amples)|(?:Amendments? to Final Protocol))", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> {
            entity.apply("DOC.11.0", "Deviation from Guidelines found", "n-a");
        });
        entityCreationService.betweenRegexes("(?:Deviations? (?:from|to)(?: the)? [S|s]tudy [P|p]lan)", "(?:Regulatory Guidelines)|(?:Other)|(?:Distribution of the report)|(?:Performing laboratory test)|(?:Distribution of the report)|(?:Retention of [S|s]amples)", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> {
            entity.apply("DOC.11.0", "Deviation from Study Plan found", "n-a");
        });
        entityCreationService.betweenStrings("Deviations from the study plan", "Regulatory Guidelines", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> {
           entity.apply("DOC.11.0", "Deviation from the study plan found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("(?>Study plan adherence)(.{1,20}deviations.{1,20} to the study plan.{0,50}\\.)\\s", "guideline_deviation", EntityType.ENTITY, 1, $section).forEach(entity -> {
           entity.apply("DOC.11.0", "Guideline deviation found in text.", "n-a");
        });
        entityCreationService.betweenStringsIncludeEnd("Deviations from the study plan", "validity of the study.", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> {
           entity.apply("DOC.11.0", "Deviation from the study plan found", "n-a");
        });
    end

rule "DOC.11.1: Guideline Deviation in text"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
        $section: Section(
            getHeadline().containsStringIgnoreCase("Introduction")
            && containsStringIgnoreCase("deviations from the protocol")
        )
    then
        entityCreationService.byRegex("There were no deviations from the protocol.{1,100}\\.\\s", "guideline_deviation", EntityType.ENTITY, $section).forEach(entity -> {
            entity.apply("DOC.11.1", "Guideline deviation found in text.", "n-a");
        });
    end


// Rule unit: DOC.12
rule "DOC.12.0: Clinical Signs"
    when
        FileAttribute(label == "OECD Number", value == "425")
        $headline: Headline(containsAnyStringIgnoreCase("Clinical Signs", "Macroscopic Findings") && !containsString("TABLE") && !getHeadline().containsStringIgnoreCase("3 - MACROSCOPIC FINDINGS"))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "clinical_signs", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.12.0", "Clinical Signs found", "n-a"));
    end


// Rule unit: DOC.13
rule "DOC.13.0: Dosages"
    when
        FileAttribute(label == "OECD Number", value == "425")
        $section: Section(
            (anyHeadlineContainsStringIgnoreCase("Dosages") || anyHeadlineContainsStringIgnoreCase("Study Design"))
            && !getHeadline().containsString("TABLE")
        )
    then
        entityCreationService.betweenStringsIncludeStartAndEnd("The animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> {
            entity.apply("DOC.13.0", "Dosage found", "n-a");
        });
        entityCreationService.betweenStringsIncludeStartAndEnd("Animals were treated", ".", "dosages", EntityType.ENTITY, $section).forEach(entity -> {
            entity.apply("DOC.13.0", "Dosage found", "n-a");
        });
        entityCreationService.byRegexWithLineBreaks("(?:\\.[\\s|\\n]|^.{5,20}\\n)([^\\.]{1,200}(?:animal|given|received)[^\\.]{1,200}dose\\s(?:levels?\\s)?(?:of|at)[^\\.]{1,200})(?:\\.[\\s|\\n|$])", "dosages", EntityType.ENTITY,1, $section).forEach(entity -> {
            entity.apply("DOC.13.0", "Dosage found", "n-a");
        });
    end


// Rule unit: DOC.14
rule "DOC.14.0: Mortality"
    when
        $headline: Headline(containsString("Mortality") && !containsString("TABLE"))
        FileAttribute(label == "OECD Number", value == "425")
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.14.0", "Mortality found", "n-a"));
    end


// Rule unit: DOC.15
rule "DOC.15.0: Study Conclusion"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","471"))
            $section: Section(
                getHeadline().containsStringIgnoreCase("Conclusion")
            )
    then
         entityCreationService.bySemanticNodeParagraphsOnly($section, "study_conclusion", EntityType.ENTITY)
         .forEach(entity -> entity.apply("DOC.15.0", "Study Conclusion found", "n-a"));
    end


// Rule unit: DOC.16
rule "DOC.16.0: Weight Behavior Changes"
    when
    FileAttribute(label == "OECD Number", value == "402")
        $section: Section(
            getHeadline().containsString("Results")
            && (
                containsString("body weight")
                || containsString("body weights")
                || containsString("bodyweight")
                || containsString("bodyweights")
            )
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "weight_behavior_changes", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.16.0", "Weight behavior changes found", "n-a"));
    end


// Rule unit: DOC.17
rule "DOC.17.0: Necropsy findings"
    when
    FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","436"))
        $section: Section(
            (
                anyHeadlineContainsStringIgnoreCase("Necropsy")
                || getHeadline().containsStringIgnoreCase("Macroscopic Findings")
                || getHeadline().containsStringIgnoreCase("Macroscopic examination")
            )
            && !getHeadline().containsStringIgnoreCase("Table")
            && !getHeadline().containsStringIgnoreCase("Appendix")
            && !getHeadline().containsStringIgnoreCase("3 - MACROSCOPIC FINDINGS")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "necropsy_findings", EntityType.ENTITY)
            .forEach( entity -> entity.apply("DOC.17.0", "Necropsy section found", "n-a"));
    end


// Rule unit: DOC.18
rule "DOC.18.0: Clinical observations"
    when
        FileAttribute(label == "OECD Number", value == "403")
        $section: Section(
             (
                anyHeadlineContainsStringIgnoreCase("Clinical Observations")
                || anyHeadlineContainsStringIgnoreCase("Clinical observations")
                || anyHeadlineContainsStringIgnoreCase("In-life Observations")
                || anyHeadlineContainsStringIgnoreCase("Postmortem Observations")
                )
            && !anyHeadlineContainsStringIgnoreCase("Appendix")
            && !anyHeadlineContainsStringIgnoreCase("Table")
            && !anyHeadlineContainsStringIgnoreCase("Mortality")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "clinical_observations", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.18.0", "Clinical observations section found", "n-a"));
    end


// Rule unit: DOC.19
rule "DOC.19.0: Bodyweight changes"
    when
        FileAttribute(label == "OECD Number", value == "403")
        $headline: Headline(containsAnyStringIgnoreCase("Bodyweight", "Bodyweights", "Body Weights", "Body Weight"), !containsAnyStringIgnoreCase("Appendix", "TABLE"))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "bodyweight_changes", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.19.0", "Bodyweight section found", "n-a"));
    end


// Rule unit: DOC.20
rule "DOC.20.0: Study Design"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","404","405","406","428","429","438","439","474","487"))
        $section: Section(
            anyHeadlineContainsStringIgnoreCase("study design")
            && !anyHeadlineContainsString("Preliminary screening test")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "study_design", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.20.0", "Study design section found", "n-a"));
    end

rule "DOC.20.1: Study Design"
    when
        Headline(containsStringIgnoreCase("Study Design"), $sectionIdentifier: getSectionIdentifier())
        $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "study_design", EntityType.ENTITY)
            .forEach(entity -> {
                entity.apply("DOC.20.1", "Study design section found", "n-a");
            });
    end


// Rule unit: DOC.21
rule "DOC.21.0: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487"))
         $parentHeadline: Headline(
              containsAnyString("Results", "Conclusion"),
              !containsAnyString("POSITIVE CONTROL", "Positive Control", "Evaluation", "Micronucleus", "TABLE", "DISCUSSION", "CONCLUSIONS", "Interpretation","Viability", "analysis"),
              $sectionIdentifier: getSectionIdentifier()
          )
         not Headline(getSectionIdentifier().isChildOf($sectionIdentifier))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($parentHeadline.getParent(), "results_and_conclusion", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.21.0", "Results and Conclusion found", "n-a"));
    end

rule "DOC.21.1: Results and Conclusion (406, 428, 438, 439, 474 & 487)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("406","428","438","439","474","487"))
         Headline(
              containsAnyString("Results", "Conclusion"),
              !containsAnyString("POSITIVE CONTROL", "Positive Control", "Evaluation", "Micronucleus", "TABLE", "DISCUSSION", "CONCLUSIONS", "Interpretation","Viability", "analysis"),
              $sectionIdentifier: getSectionIdentifier()
          )
         $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifier))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "results_and_conclusion", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.21.1", "Results and Conclusion found", "n-a"));
    end


// Rule unit: DOC.22
rule "DOC.22.0: Detailing (404 & 405)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("404","405"))
        $section: Section(
            anyHeadlineContainsStringIgnoreCase("Results")
            && !getHeadline().containsStringIgnoreCase("Evaluation")
            && !getHeadline().containsStringIgnoreCase("study")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "detailing", EntityType.ENTITY)
                    .forEach(entity -> entity.apply("DOC.22.0", "Detailing found", "n-a"));
    end


// Rule unit: DOC.23
rule "DOC.23.0: Preliminary Test Results (429)"
    when
        FileAttribute(label == "OECD Number", value == "429")
        $section: Section(
            ((anyHeadlineContainsString("Preliminary Screening Test") && containsString("Clinical observations"))
            || anyHeadlineContainsString("Pre-Experiment"))
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "preliminary_test_results", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.23.0", "Preliminary Test Results found", "n-a"));
    end


// Rule unit: DOC.24
rule "DOC.24.0: Test Results (429)"
    when
        FileAttribute(label == "OECD Number", value == "429")
        $section: Section((getHeadline().containsString("RESULTS AND DISCUSSION") || getHeadline().containsString("Estimation of the proliferative response of lymph node cells") || getHeadline().containsString("Results in the Main Experiment")))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "test_results", EntityType.ENTITY)
        .forEach(entity -> entity.apply("DOC.24.0", "Test Results found", "n-a"));
    end

rule "DOC.24.1: Test Results (429)"
    when
        Headline(containsStringIgnoreCase("RESULTS AND DISCUSSION"), $sectionIdentifierResultsAndDiscussion: getSectionIdentifier())
        $headline: Headline(getSectionIdentifier().isChildOf($sectionIdentifierResultsAndDiscussion))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "test_results", EntityType.ENTITY)
            .forEach(entity -> {
                entity.apply("DOC.24.1", "Test Results found", "n-a");
            });
    end


// Rule unit: DOC.25
rule "DOC.25.0: Approach used (429)"
    when
        FileAttribute(label == "OECD Number", value == "429")
        $section: Section(
            hasEntitiesOfType("species")
            && (containsStringIgnoreCase("animals per") || containsStringIgnoreCase("animals /"))
        )
    then
        entityCreationService.byRegexIgnoreCase("\\banimals (?:per|\\/) .{0,15}(group)\\b", "approach_used", EntityType.ENTITY,1, $section).forEach(entity -> {
            entity.apply("DOC.25.0", "Study animal approach found.", "n-a");
        });
    end


// Rule unit: DOC.26
rule "DOC.26.0: Sex"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("405","429"))
        $section: Section(
            (
                anyHeadlineContainsStringIgnoreCase("animal")
                || anyHeadlineContainsStringIgnoreCase("test system")
            )
            && !getHeadline().containsStringIgnoreCase("selection")
            && (
                containsStringIgnoreCase("sex:")
                || containsStringIgnoreCase("male")
                || containsStringIgnoreCase("female")
            )
        )
    then
        entityCreationService.byRegexIgnoreCase("([S|s]ex:)?[\\w\\s]{0,10}\\b(males?|females?)\\b", "sex", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.26.0", "Test animal sex found", "n-a");
        });
    end


// Rule unit: DOC.27
rule "DOC.27.0: Animal Number 405"
    when
        FileAttribute(label == "OECD Number", value == "405")
        $section: Section(
             (
                anyHeadlineContainsStringIgnoreCase("animal")
                || anyHeadlineContainsStringIgnoreCase("test system")
                || anyHeadlineContainsStringIgnoreCase("reaction")
            )
            && !getHeadline().containsString("selection")
            && (
                containsStringIgnoreCase("number of animals")
                || containsStringIgnoreCase("no.")
            )
        )
    then
         entityCreationService.byRegexIgnoreCase("(Number of animals:)[\\w\\s]{0,10}\\b([\\d]{1,3})\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.27.0", "Number of animals found", "n-a");
         });
         entityCreationService.byRegexIgnoreCase("(?:.{1,10} No\\. )([\\d\\w\\-]{3,8})", "animal_numbers", EntityType.ENTITY,1, $section).forEach(entity -> {
            entity.apply("DOC.27.0", "Number of animals found", "n-a");
         });
    end


// Rule unit: DOC.28
rule "DOC.28.0: Animal Number 429"
    when
     FileAttribute(label == "OECD Number", value == "429")
        $section: Section(
            (
                anyHeadlineContainsStringIgnoreCase("animal")
                || anyHeadlineContainsStringIgnoreCase("test system")
            )
            && !getHeadline().containsString("selection")
            && containsStringIgnoreCase("number of animals")
            && (containsStringIgnoreCase("per") || containsString("/"))
            && containsStringIgnoreCase("group")
        )
    then
        entityCreationService.byRegexIgnoreCase("(Number of animals per group:)[\\w\\s]{0,10}\\b([\\d]{1,3})\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.28.0", "Number of animals in group found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("(Number of animals per group:).{0,60}\\b([\\d]{1,3})\\sper group\\b", "number_of_animals", EntityType.ENTITY,2, $section).forEach(entity -> {
            entity.apply("DOC.28.0", "Number of animals in group found", "n-a");
        });
        entityCreationService.byRegexIgnoreCase("([\\d]{1,3})[\\w\\s\\/]{0,20}(?:treatment )?group\\b", "number_of_animals", EntityType.ENTITY,1 , $section).forEach(entity -> {
            entity.apply("DOC.28.0", "Number of animals in group found", "n-a");
        });
    end

rule "DOC.28.1: No. Of animals - Fallback to appendix tables listing all individual animals for 429"
    when
        $keyword: String() from List.of("Animal Number", "Animal No.", "Animal number")
        $table: Table(containsString($keyword) && getHeadline().containsString("TABLE") && getHeadline().containsString("Individual"))
        FileAttribute(label == "OECD Number", value == "429")
    then
        $table.streamTableCellsWithHeader($keyword)
                .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "animal_numbers", EntityType.ENTITY))
                .filter(Optional::isPresent)
                .map(Optional::get)
                .forEach(entity -> {
                    entity.apply("DOC.28.1", "Animal number found.", "n-a");
                });
     end


// Rule unit: DOC.29
rule "DOC.29.0: 4h Exposure"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("403","436"))
        $section: Section(
            (containsStringIgnoreCase("4 hours") || containsStringIgnoreCase("four hours"))
        )
    then
        entityCreationService.byRegexIgnoreCase("((?<=\\.\\s\\b).{1,100}(4|four) hours.*?\\.) ", "4h_exposure", EntityType.ENTITY,1, $section).forEach(entity -> {
           entity.apply("DOC.29.0", "4h exposure sentence found", "n-a");
        });
    end


// Rule unit: DOC.30
rule "DOC.30.0: Dilution of the test substance"
    when
        FileAttribute(label == "OECD Number", value == "404")
        $section: Section(
             getHeadline().containsString("Formulation")
            && containsString("dilution")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "dilution", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.30.0", "Dilution found.", "n-a"));
    end


// Rule unit: DOC.31
rule "DOC.31.0: Positive Control"
    when
        FileAttribute(label == "OECD Number", value == "429")
        $section: Section(
            getHeadline().containsStringIgnoreCase("Positive Control")
            && !(getHeadline().containsStringIgnoreCase("Appendix") || getHeadline().containsStringIgnoreCase("Table"))
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "positive_control", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.31.0", "Positive control found.", "n-a"));
    end


// Rule unit: DOC.32
rule "DOC.32.0: Mortality Statement"
    when
        FileAttribute(label == "OECD Number", value == "402")
        $headline: Headline(containsStringIgnoreCase("Mortality") && !containsString("TABLE"))
    then
        entityCreationService.bySemanticNodeParagraphsOnly($headline.getParent(), "mortality_statement", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.32.0", "Mortality Statement found", "n-a"));
    end


// Rule unit: DOC.33
rule "DOC.33.0: Dose Mortality"
    when
        FileAttribute(label == "OECD Number", value == "425")
        $table: Table(
            (hasHeader("Mortality") || hasHeader("Long Term Results") || hasHeader("LongTerm Outcome") || hasHeader("Long Term Outcome") || hasHeader("Comments") || hasHeader("Viability / Mortality") || hasHeader("Viability/Mortality"))
            &&
            (hasHeader("Dose [mg/kg bodyweight]") || hasHeader("Dose [mg/kg body weight]") ||hasHeader("Dose (mg/kg)") || hasHeader("Dose levei (mg/kg)") || hasHeader("Dose Level (mg/kg)") || hasHeader("Dose level (mg/kg)") || hasHeader("Dosage [mg/kg body weight]"))
        )
    then
        Stream.of($table.streamTableCellsWithHeader("Mortality"),
                            $table.streamTableCellsWithHeader("Comments"),
                            $table.streamTableCellsWithHeader("Long Term Results"),
                            $table.streamTableCellsWithHeader("Long Term Outcome"),
                            $table.streamTableCellsWithHeader("LongTerm Outcome"),
                            $table.streamTableCellsWithHeader("Viability / Mortality"),
                            $table.streamTableCellsWithHeader("Viability/Mortality")
                            ).flatMap(a -> a)
                            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
                            .filter(Optional::isPresent)
                            .map(Optional::get)
                            .forEach(entity -> {
                                entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
                            });

        Stream.of($table.streamTableCellsWithHeader("Dose [mg/kg bodyweight]"),
                            $table.streamTableCellsWithHeader("Dose [mg/kg body weight]"),
                            $table.streamTableCellsWithHeader("Dose levei (mg/kg)"),
                            $table.streamTableCellsWithHeader("Dose Level (mg/kg)"),
                            $table.streamTableCellsWithHeader("Dose level (mg/kg)"),
                            $table.streamTableCellsWithHeader("Dose (mg/kg)"),
                            $table.streamTableCellsWithHeader("Dosage [mg/kg body weight]")
                            ).flatMap(a -> a)
                            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
                            .filter(Optional::isPresent)
                            .map(Optional::get)
                            .forEach(entity -> {
                                entity.apply("DOC.33.0", "Dose Mortality Data found.", "n-a");
                            });
    end


// Rule unit: DOC.34
rule "DOC.34.0: Results (Main Study)"
    when
        FileAttribute(label == "OECD Number", value == "429")
        $section: Section(
            getHeadline().containsString("Results")
            && getHeadline().getTextRange().length() < 20
            && !(getHeadline().containsString("Appendix") || getHeadline().containsString("Table"))
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "results_(main_study)", EntityType.ENTITY)
                            .forEach(entity -> entity.apply("DOC.34.0", "Results for main study found.", "n-a"));
    end


// Rule unit: DOC.35
rule "DOC.35.0: Doses (mg/kg bodyweight)"
    when
        FileAttribute(label == "OECD Number", value == "402")
        $section: Section(
             anyHeadlineContainsStringIgnoreCase("study design")
        )
    then
        entityCreationService.bySemanticNodeParagraphsOnly($section, "doses_(mg_kg_bw)", EntityType.ENTITY)
            .forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
    end

//------------------------------------ Table extraction rules ------------------------------------

// Rule unit: TAB.0
rule "TAB.0.0: Study Type File Attribute"
    when
        not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
            && containsAnyString("OECD", "EPA", "OPPTS"))
    then
        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
            .forEach(fileAttribute -> insert(fileAttribute));
        RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
            .map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
            .map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
            .forEach(fileAttribute -> insert(fileAttribute));
    end

rule "TAB.0.1: Guidelines"
    when
        $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
    then
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
        entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
        entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
        entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
            .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
    end


// Rule unit: TAB.1
rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Full Table"))
        $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
        $tableCell: TableCell(!header) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
    end


// Rule unit: TAB.2
rule "TAB.2.0: Individual row extraction (Clinical Signs)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Individual Rows Extraction"))
        $table: Table(hasHeader("Animal No."), (hasRowWithHeaderAndAnyValue("Animal No.", List.of("120-2", "120-5")))) from $section.streamChildren().toList()
        TableCell($row: row, containsAnyString("120-2", "120-5")) from $table.streamTableCellsWithHeader("Animal No.").toList()
        $tableCell: TableCell($row == row) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($tableCell, "clinical_signs", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.2.0", "Individual row based on animal number"));
    end


// Rule unit: TAB.3
rule "TAB.3.0: Individual column extraction (Strain)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Individual Column"))
        $table: Table(hasHeader("Sex")) from $section.streamChildren().toList()
    then
        $table.streamTableCellsWithHeader("Sex")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dosages", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.3.0", "Individual column based on column header"));
    end


// Rule unit: TAB.4
rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Combined Columns"))
        $table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
    then
        $table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
    end

rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Combined Columns"))
        $table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
    then
        $table.streamTableCellsWithHeader("Mortality")
            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
            .filter(Optional::isPresent)
            .map(Optional::get)
            .forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
    end


// Rule unit: TAB.5
rule "TAB.5.0: Targeted cell extraction"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Value Extraction"))
        $table: Table(hasHeader("Mortality"), hasRowWithHeaderAndAnyValue("Sex", List.of("male", "Male")), hasRowWithHeaderAndValue("Mortality", "Survived")) from $section.streamChildren().toList()
        TableCell(containsWordIgnoreCase("Male"), $row: row) from $table.streamTableCellsWithHeader("Sex").toList()
        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Mortality").toList()
        $dosageCell: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
    then
        entityCreationService.bySemanticNode($dosageCell,"doses_mg_kg_bw", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.5.0", "Dosage found in row with survived male"));
    end


// Rule unit: TAB.6
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
    when
        $section: Section(getHeadline().containsString("Advanced Table Extraction"), containsAllStrings("female", "Female", "Survived", "Group 2"))
        $table: Table(hasHeader("Group 2")) from $section.streamChildren().toList()
        TableCell(containsWordIgnoreCase("Female"), $row: row) from $table.streamTableCellsWithHeader("Group 2").toList()
        TableCell($row == row, containsStringIgnoreCase("Survived")) from $table.streamTableCellsWithHeader("Group 2").toList()
        $femaleSurvived: TableCell($row == row) from $table.streamTableCellsWithHeader("Group 2").toList()
    then
        entityCreationService.bySemanticNode($femaleSurvived, "experiment_female_survived", EntityType.ENTITY)
            .ifPresent(entity -> entity.apply("TAB.6.0", "Female in group to experimental start date"));
    end


// Rule unit: TAB.7
rule "TAB.7.0: Indicator (Species)"
    when
        FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
        $section: Section(getHeadline().containsString("Entity-Based"))
        $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
        TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
        TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
        $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
    then
        entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
            .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
    end

//------------------------------------ Manual redaction rules ------------------------------------

// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $entityToBeResized: TextEntity(matchesAnnotationId($id))
    then
        manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
        retract($resizeRedaction);
    end

rule "MAN.0.1: Apply manual resize redaction"
    salience 128
    when
        $resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
        not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
        $imageToBeResized: Image(id == $id)
    then
        manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
        retract($resizeRedaction);
    end


// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
        $entityToBeRemoved: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeRemoved.addManualChange($idRemoval);
        retract($idRemoval);
    end

rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
    salience 128
    when
        $idRemoval: IdRemoval($id: annotationId)
        $imageEntityToBeRemoved: Image($id == id)
    then
        $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
        retract($idRemoval);
    end


// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $entityToForce: TextEntity(matchesAnnotationId($id))
    then
        $entityToForce.addManualChange($force);
        retract($force);
    end

rule "MAN.2.1: Apply force redaction to images"
    salience 128
    when
        $force: ManualForceRedaction($id: annotationId)
        $imageToForce: Image(id == $id)
    then
        $imageToForce.getManualOverwrite().addChange($force);
        retract($force);
    end


// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
    then
        $entityToBeRecategorized.addManualChange($recategorization);
        retract($recategorization);
    end

rule "MAN.3.1: Apply entity recategorization of same type"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
    then
        $entityToBeRecategorized.addManualChange($recategorization);
        retract($recategorization);
    end

rule "MAN.3.2: Apply image recategorization"
    salience 128
    when
        $recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
        not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
        $imageToBeRecategorized: Image($id == id)
    then
        manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
        retract($recategorization);
    end

rule "MAN.3.3: Apply recategorization entities by default"
    salience 128
    when
        $entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
    then
        $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
    end

// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $imageToBeRecategorized: Image($id == id)
    then
        $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
        retract($legalBasisChange)
    end

rule "MAN.4.1: Apply legal basis change"
    salience 128
    when
        $legalBasisChange: ManualLegalBasisChange($id: annotationId)
        $entityToBeChanged: TextEntity(matchesAnnotationId($id))
    then
        $entityToBeChanged.addManualChange($legalBasisChange);
        retract($legalBasisChange)
    end


//------------------------------------ Entity merging rules ------------------------------------

// Rule unit: X.0
rule "X.0.0: Remove Entity contained by Entity of same type"
    salience 65
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.type() == $contained.type(),
            $container.entityType == $contained.entityType,
            $container != $contained,
            !$container.removed(),
            !$container.hasManualChanges(),
            !$contained.hasManualChanges(),
            !$contained.removed()
        )
        not TextEntity(
            getTextRange().equals($container.getTextRange()),
            type() == $container.type(),
            entityType == EntityType.DICTIONARY_REMOVAL,
            engines contains Engine.DOSSIER_DICTIONARY,
            !hasManualChanges()
        )
    then
        $contained.remove("X.0.0", "remove Entity contained by Entity of same type");
end

rule "X.0.1: Remove Entity contained by Entity of same type with manual changes"
    salience 65
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.type() == $contained.type(),
            $container.entityType == $contained.entityType,
            $container != $contained,
            !$container.removed(),
            $container.hasManualChanges(),
            !$contained.hasManualChanges(),
            !$contained.removed()
        )
    then
        $contained.remove("X.0.1", "remove Entity contained by Entity of same type with manual changes");
end


// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_POSITIVE,
            $container.active(),
            $contained.entityType == EntityType.ENTITY,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
end


rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_POSITIVE,
            $container.active(),
            $contained.entityType == EntityType.HINT,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.2.1", "remove Entity of type HINT when contained by FALSE_POSITIVE");
end


rule "X.3.0: Remove RECOMMENDATION Contained by FALSE_RECOMMENDATION"
    salience 64
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.FALSE_RECOMMENDATION,
            $container.active(),
            $contained.entityType == EntityType.RECOMMENDATION,
            $contained.type() == $container.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
end


// Rule unit: X.4
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
    salience 256
    when
        $equality: Equality(
            $a: a,
            $b: b,
            $a.type() == $b.type(),
            ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
            $a.active(),
            $b.entityType == EntityType.RECOMMENDATION,
            !$b.hasManualChanges()
        )
    then
        $a.addEngines($b.getEngines());
        $b.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
    end


// Rule unit: X.5
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
    salience 256
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            ($a.entityType == EntityType.ENTITY || $a.entityType == EntityType.HINT),
            $a.active(),
            $b.entityType == EntityType.RECOMMENDATION,
            !$b.hasManualChanges()
        )
    then
        $b.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
    end


// Rule unit: X.5
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
    salience 256
    when
        $containment: Containment(
            $container: container,
            $contained: contained,
            $container.entityType == EntityType.RECOMMENDATION,
            $container.active(),
            $contained.entityType == EntityType.RECOMMENDATION,
            $container.type() != $contained.type(),
            !$contained.hasManualChanges()
        )
    then
        $contained.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
end


// Rule unit: X.7
rule "X.7.0: Remove all images"
    salience 512
    when
        $image: Image(imageType != ImageType.OCR, !hasManualChanges())
    then
        $image.remove("X.7.0", "remove all images");
        retract($image);
    end


// Rule unit: X.8
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
    salience 257
    when
        $equality: Equality(
            $a: a,
            $b: b,
            $a.type() == $b.type(),
            $a.engines contains Engine.IMPORTED,
            $a.active(),
            $b.engines not contains Engine.IMPORTED,
            $a != $b
        )
    then
        $b.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
        $a.addEngines($b.getEngines());
    end

rule "X.8.1: Remove Entity when intersected by imported Entity"
    salience 256
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            $a.engines contains Engine.IMPORTED,
            $a.active(),
            $b.engines not contains Engine.IMPORTED,
            $a != $b
        )
    then
        $b.remove("X.8.1", "remove Entity when intersected by imported Entity");
    end

// Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures"
    when
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
        $signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
    then
        $aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
        $signature.addEngine(LayoutEngine.AI);
    end

// Rule unit: X.10
rule "X.10.0: remove false positives of ai"
    when
        $anyImage: Image(engines contains LayoutEngine.ALGORITHM)
        $aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
    then
        $aiSignature.remove("X.10.0", "Removed because false positive");
    end


// Rule unit: X.11
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
    salience 64
    when
        $intersection: Intersection(
            $a: a,
            $b: b,
            $a.engines contains Engine.MANUAL,
            $a.active(),
            $b.dictionaryEntry,
            $b.engines not contains Engine.MANUAL
        )
    then
        $b.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
    end


//------------------------------------ Dictionary merging rules ------------------------------------

// Rule unit: DICT.0
rule "DICT.0.0: Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
    salience 64
    when
        $dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY)
        $entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
    then
        $entity.remove("DICT.0.0", "Remove Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
        $entity.addEngine(Engine.DOSSIER_DICTIONARY);
    end


//------------------------------------ File attributes rules ------------------------------------

// Rule unit: FA.1
rule "FA.1.0: Remove duplicate FileAttributes"
    salience 64
    when
        $fileAttribute: FileAttribute($label: label, $value: value)
        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
    then
        retract($duplicate);
    end


//------------------------------------ Local dictionary search rules ------------------------------------

// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
    agenda-group "LOCAL_DICTIONARY_ADDS"
    salience -999
    when
        $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
    then
        entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
            .forEach(entity ->  {
                Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
                matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
            });
    end