redaction-service/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl

package drools

import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityEnrichmentService.setFields;


import java.util.List;
import java.util.LinkedList;
import java.util.HashSet;

import com.iqser.red.service.redaction.v1.server.document.graph.*
import com.iqser.red.service.redaction.v1.server.document.graph.nodes.*
import com.iqser.red.service.redaction.v1.server.document.graph.entity.*
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.*
import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import java.util.Set
import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine
import com.iqser.red.service.redaction.v1.server.document.services.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.redaction.model.Dictionary;

global DocumentGraph document
global EntityCreationService entityCreationService
global Dictionary dictionary

// --------------------------------------- CBI rules -------------------------------------------------------------------

rule "1: Redact CBI Authors (Non vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
    then
        $entity.setRedaction(true);
        setFields($entity, 1, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end

rule "2: Redact CBI Authors (Vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
        $entity: EntityNode(type == "CBI_author", entityType == EntityType.ENTITY)
    then
        $entity.setRedaction(true);
        setFields($entity, 2, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end

rule "3: Don't redact CBI Address (Non vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "no")
        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
    then
        setFields($entity, 3, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end

rule "4: Redact CBI Address (Vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $entity: EntityNode(type == "CBI_address", entityType == EntityType.ENTITY)
    then
        $entity.setRedaction(true);
        setFields($entity, 4, "Address found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end

rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"

    when
        $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
    then
        EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
        setFields($entity, 5, "Genitive Author", null, Engine.RULE);
        insert(entity);
    end


rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and add Authorname as Recommendation"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
        authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
    then
        EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
        setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
        insert(entity);
        dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
    end

rule "7: Add CBI_author with \"et al.\" Regex"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
         $section: SectionNode(containsString("et al."))
    then
         Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
         setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
         entities.forEach(entity -> insert(entity));
         entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
    end

rule "8: Add recommendation for Addresses in Test Organism sections"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
    then
        Set<EntityNode> entities = entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section);
        setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
    end


rule "9: Add recommendation for Addresses in Test Animals sections"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $section: SectionNode(containsString("Species:") && containsString("Source:"))
    then
        Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
        setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
    end

rule "12: Recommend CTL/BL laboratory that start with BL or CTL"

    when
        $section : SectionNode(containsString("BL") || containsString("CT"))
    then
        Set<EntityNode> entities = entityCreationService.byRegex("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section);
        setFields(entities, 12, "Syngenta Specific Laboratory keyword", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
    end


// --------------------------------------- PII rules -------------------------------------------------------------------


rule "10: Redacted PII Personal Identification Information (Non vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
    then
        $entity.setRedaction(true);
        setFields($entity, 10, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end



rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
    no-loop true

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $entity: EntityNode(type == "PII", entityType == EntityType.ENTITY)
    then
        $entity.setRedaction(true);
        setFields($entity, 11, "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002", null);
        update($entity)
    end

rule "12: Redact Emails by RegEx (Non vertebrate study)"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
        $section: SectionNode(containsString("@"))
    then
        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
        setFields(entities, 12, "Found by email regex", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
    end

rule "13: Redact Emails by RegEx (Vertebrate study)"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $section: SectionNode(containsString("@"))
    then
        Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
        setFields(entities, 13, "Found by email regex", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
    end


rule "14: Redact line after contact information (Non vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
        $string: String() from List.of("Contact point:",
                                         "Contact:",
                                         "Alternative contact:",
                                         "European contact:",
                                         "No:",
                                         "Contact:",
                                         "Tel.:",
                                         "Tel:",
                                         "Telephone number:",
                                         "Telephone No:",
                                         "Telephone:",
                                         "Phone No:",
                                         "Phone:",
                                         "Fax number:",
                                         "Fax:",
                                         "E-mail:",
                                         "Email:",
                                         "e-mail:",
                                         "E-mail address:")
        $section: SectionNode(containsString($string))
    then

        Set<EntityNode> entities = new HashSet<>();
        entities.addAll(entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section));
        setFields(entities, 14, "Found by contacts keyword", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
    end


rule "15: Redact line after contact information (Vertebrate study)"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $string: String() from List.of("Contact point:",
                                         "Contact:",
                                         "Alternative contact:",
                                         "European contact:",
                                         "No:",
                                         "Contact:",
                                         "Tel.:",
                                         "Tel:",
                                         "Telephone number:",
                                         "Telephone No:",
                                         "Telephone:",
                                         "Phone No:",
                                         "Phone:",
                                         "Fax number:",
                                         "Fax:",
                                         "E-mail:",
                                         "Email:",
                                         "e-mail:",
                                         "E-mail address:")
        $section: SectionNode(containsString($string))
    then
        Set<EntityNode> entities = entityCreationService.lineAfterString($string, "PII", EntityType.ENTITY, $section);
        setFields(entities, 15, "Found by contacts keyword", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
    end


rule "16: redact line between contact keywords"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
    then
        Set<EntityNode> entities = new HashSet<>();
        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
        setFields(entities, 16, "Found between contacts keyword", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
    end

rule "17: redact line between contact keywords"
    agenda-group "LOCAL_DICTIONARY_ADDS"

    when
        FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
        $section: SectionNode((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
    then
        Set<EntityNode> entities = new HashSet<>();
        entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
        entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section));
        setFields(entities, 17, "Found between contacts keyword", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
        entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
    end

rule "18: Redact Phone and Fax by RegEx"

    when
        $section: SectionNode(containsString("Contact") ||
            containsString("Telephone") ||
            containsString("Phone") ||
            containsString("Fax") ||
            containsString("Tel") ||
            containsString("Ter") ||
            containsString("Mobile") ||
            containsString("Fel") ||
            containsString("Fer"))
    then
        Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
        setFields(entities, 18, "Found by Phone and Fax regex", null, Engine.RULE);
        entities.forEach(entity -> insert(entity));
    end