RED-6009 - Document Tree Structure

* updated rules to production syngenta rules

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/entity/RedactionEntity.java

@@ -123,7 +123,6 @@ public class RedactionEntity {
     public void addMatchedRule(int ruleNumber) {
 
         matchedRules.add(ruleNumber);
-        engines.add(Engine.RULE);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/graph/nodes/TableNode.java

@@ -78,10 +78,10 @@ public class TableNode implements SemanticNode {
     }
 
 
-    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(String type) {
+    public Stream<RedactionEntity> streamEntitiesWhereRowContainsEntitiesOfType(List<String> types) {
 
         List<Integer> rowsWithEntityOfType = getEntities().stream()
-                .filter(redactionEntity -> redactionEntity.getType().equals(type))
+                .filter(redactionEntity -> types.stream().anyMatch(type -> type.equals(redactionEntity.getType())))
                 .map(RedactionEntity::getIntersectingNodes)
                 .filter(node -> node instanceof TableCellNode)
                 .map(node -> (TableCellNode) node)
@@ -143,7 +143,7 @@ public class TableNode implements SemanticNode {
 
     public boolean hasHeader(String header) {
 
-        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().contains(header));
+        return streamHeaders().anyMatch(tableCellNode -> tableCellNode.buildTextBlock().getSearchText().strip().equals(header));
     }
 
 
@@ -159,6 +159,19 @@ public class TableNode implements SemanticNode {
     }
 
 
+    public List<RedactionEntity> getEntitiesOfTypeInSameRow(String type, RedactionEntity redactionEntity) {
+
+        return redactionEntity.getIntersectingNodes()
+                .stream()
+                .filter(node -> node instanceof TableCellNode)
+                .map(node -> (TableCellNode) node)
+                .flatMap(tableCellNode -> streamRow(tableCellNode.getRow()))
+                .map(cell -> cell.getEntitiesOfType(type))
+                .flatMap(Collection::stream)
+                .toList();
+    }
+
+
     @Override
     public TextBlock buildTextBlock() {
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/services/EntityCreationService.java

@@ -100,7 +100,27 @@ public class EntityCreationService {
 
     public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
 
-        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+        return byRegex(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, SemanticNode node) {
+
+        return byRegexIgnoreCase(regexPattern, type, entityType, 0, node);
+    }
+
+
+    public Stream<RedactionEntity> byRegex(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegex(regexPattern, group, node.buildTextBlock()).stream().map(boundary -> byBoundary(boundary, type, entityType, node));
+    }
+
+
+    public Stream<RedactionEntity> byRegexIgnoreCase(String regexPattern, String type, EntityType entityType, int group, SemanticNode node) {
+
+        return RedactionSearchUtility.findBoundariesByRegexCaseInsensitive(regexPattern, group, node.buildTextBlock())
+                .stream()
+                .map(boundary -> byBoundary(boundary, type, entityType, node));
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/layoutparsing/document/utils/RedactionSearchUtility.java

@@ -74,26 +74,34 @@ public class RedactionSearchUtility {
     }
     
 
-    public static List<Boundary> findBoundariesByRegex(String regexPattern, CharSequence searchText) {
-
-        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
-        Matcher matcher = pattern.matcher(searchText);
-        List<Boundary> boundaries = new LinkedList<>();
-        while (matcher.find()) {
-            boundaries.add(new Boundary(matcher.start(), matcher.end()));
-        }
-        return boundaries;
-
-    }
-
-
     public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
 
         Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return getBoundariesByPattern(textBlock, 0, pattern);
+
+    }
+
+
+    public static List<Boundary> findBoundariesByRegex(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
+        return getBoundariesByPattern(textBlock, group, pattern);
+    }
+
+
+    public static List<Boundary> findBoundariesByRegexCaseInsensitive(String regexPattern, int group, TextBlock textBlock) {
+
+        Pattern pattern = Patterns.getCompiledPattern(regexPattern, true);
+        return getBoundariesByPattern(textBlock, group, pattern);
+    }
+
+
+    private static List<Boundary> getBoundariesByPattern(TextBlock textBlock, int group, Pattern pattern) {
+
         Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
         List<Boundary> boundaries = new LinkedList<>();
         while (matcher.find()) {
-            boundaries.add(new Boundary(matcher.start() + textBlock.getBoundary().start(), matcher.end() + textBlock.getBoundary().start()));
+            boundaries.add(new Boundary(matcher.start(group) + textBlock.getBoundary().start(), matcher.end(group) + textBlock.getBoundary().start()));
         }
         return boundaries;
     }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -1375,7 +1375,7 @@ public class Section {
 
                 if (addAsRecommendations && !isLocal()) {
                     String cleanedWord = word.replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
-                    Pattern pattern = Patterns.AUTHOR_TABLE_SPITTER;
+                    Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
                     Matcher matcher = pattern.matcher(cleanedWord);
 
                     while (matcher.find()) {

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/dictionary/Dictionary.java

@@ -6,10 +6,14 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.lang3.StringUtils;
 
 import com.iqser.red.service.redaction.v1.server.exception.NotFoundException;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
 
 import lombok.Data;
 import lombok.Getter;
@@ -102,4 +106,25 @@ public class Dictionary {
         }
     }
 
+
+    public void addLocalDictionaryEntry(RedactionEntity redactionEntity) {
+
+        addLocalDictionaryEntry(redactionEntity.getType(), redactionEntity.getValue(), false);
+    }
+
+
+    public void addMultipleAuthorsAsRecommendation(RedactionEntity redactionEntity) {
+
+        String cleanedWord = redactionEntity.getValue().replaceAll(",", " ").replaceAll("  ", " ").trim() + " ";
+        Pattern pattern = Patterns.AUTHOR_TABLE_SPLITTER;
+        Matcher matcher = pattern.matcher(cleanedWord);
+
+        while (matcher.find()) {
+            String match = matcher.group().trim();
+            if (match.length() >= 3) {
+                addLocalDictionaryEntry(redactionEntity.getType(), match, true);
+            }
+        }
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/utils/Patterns.java

@@ -11,7 +11,7 @@ public final class Patterns {
 
     public static Map<String, Pattern> patternCache = new HashMap<>();
 
-    public static Pattern AUTHOR_TABLE_SPITTER = Pattern.compile(
+    public static Pattern AUTHOR_TABLE_SPLITTER = Pattern.compile(
             "(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2}\\.){1,3})|(((((di)|(van)) )|[A-Z]’)?[A-ZÄÖÜ][\\wäöüéèê]{2,500}( ?[A-ZÄÖÜ]{1,2} ){1,3})");
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationV2Test.java

@@ -131,9 +131,8 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
-        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
         assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
-
     }
 
 
@@ -170,7 +169,7 @@ public class RedactionIntegrationV2Test extends AbstractRedactionIntegrationTest
         assertThat(redactionLogEntry.isExcluded()).isEqualTo(false);
         assertThat(redactionLogEntry.isDictionaryEntry()).isEqualTo(true);
 
-        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(2);
+        assertThat(redactionLogEntry.getEngines().size()).isEqualTo(1);
         assertThat(redactionLogEntry.getEngines().contains(Engine.DICTIONARY)).isEqualTo(true);
 
     }

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/prod_syngenta_new.drl

@@ -0,0 +1,561 @@
+package drools
+
+import static java.lang.String.format;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.anyMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility.exactMatch;
+import static com.iqser.red.service.redaction.v1.server.layoutparsing.document.data.mapper.PropertiesMapper.parseImageType;
+
+import java.util.List;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.Liszt;
+import java.util.LinkedList;
+import java.util.HashSet;
+
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.nodes.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.*;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.textblock.*;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityType;
+import com.iqser.red.service.redaction.v1.server.redaction.model.ImageType;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
+import java.util.Set
+import com.iqser.red.service.persistence.service.v1.api.shared.model.redactionlog.Engine;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.EntityCreationService;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.Dictionary;
+import com.iqser.red.service.redaction.v1.server.redaction.model.dictionary.DictionaryModel;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualImageRecategorization;
+import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.services.ManualRedactionApplicationService;
+import com.iqser.red.service.redaction.v1.server.client.model.EntityRecognitionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.entity.RedactionEntity;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.graph.Boundary;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntitiesAdapter;
+import com.iqser.red.service.redaction.v1.server.redaction.adapter.NerEntities;
+import java.util.stream.Collectors;
+import java.util.Collection;
+import java.util.stream.Stream;
+import com.iqser.red.service.redaction.v1.server.layoutparsing.document.utils.RedactionSearchUtility;
+
+global DocumentGraph document
+global EntityCreationService entityCreationService
+global ManualRedactionApplicationService manualRedactionApplicationService
+global NerEntitiesAdapter nerEntitiesAdapter
+global Dictionary dictionary
+
+// --------------------------------------- queries -------------------------------------------------------------------
+
+query "getFileAttributes"
+    $fileAttribute: FileAttribute()
+    end
+
+// --------------------------------------- Syngenta specific laboratory recommendation -------------------------------------------------------------------
+
+rule "0: Recommend CTL/BL laboratory that start with BL or CTL"
+    when
+        $section: SectionNode(containsString("CT") || containsString("BL"))
+    then
+        /* Regular expression: ((\b((([Cc]T(([1ILli\/])| L|~P))|(BL))[\. ]?([\dA-Ziltphz~\/.:!]| ?[\(',][Ppi](\(e)?|([\(-?']\/))+( ?[\(\/\dA-Znasieg]+)?)\b( ?\/? ?\d+)?)|(\bCT[L1i]\b)) */
+        entityCreationService.byRegexIgnoreCase("((\\b((([Cc]T(([1ILli\\/])| L|~P))|(BL))[\\. ]?([\\dA-Ziltphz~\\/.:!]| ?[\\(',][Ppi](\\(e)?|([\\(-?']\\/))+( ?[\\(\\/\\dA-Znasieg]+)?)\\b( ?\\/? ?\\d+)?)|(\\bCT[L1i]\\b))", "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.addMatchedRule(0);
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
+    end
+// --------------------------------------- CBI Rules -------------------------------------------------------------------
+
+rule "1: Redact CBI Authors"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_author", entityType == EntityType.ENTITY)
+    then
+        $entity.setRedaction(true);
+        $entity.addMatchedRule(1);
+        $entity.setRedactionReason("Author found");
+        $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "2: Don't redact CBI Address (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
+    then
+        $entity.setRedaction(false);
+        $entity.addMatchedRule(2);
+        $entity.setRedactionReason("Address found for non vertebrate study");
+    end
+
+rule "3: Redact CBI Address (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value.toLowerCase() == "yes")
+        $entity: RedactionEntity(type == "CBI_address", entityType == EntityType.ENTITY)
+    then
+        $entity.setRedaction(true);
+        $entity.addMatchedRule(4);
+        $entity.setRedactionReason("Address found");
+        $entity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
+    end
+
+rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
+    when
+        $entity: RedactionEntity(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
+    then
+        RedactionEntity falsePositive = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
+        falsePositive.addMatchedRule(5);
+        insert(falsePositive);
+    end
+
+rule "6: Redact all Cell's with Header Author(s) as CBI_author"
+    when
+        $table: TableNode(hasHeader("Author(s)"))
+    then
+        $table.streamTableCellsWithHeader("Author(s)")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.addMatchedRule(6);
+                redactionEntity.addEngine(Engine.RULE);
+                redactionEntity.setRedactionReason("Author(s) found");
+                redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
+                insert(redactionEntity);
+            });
+    end
+
+rule "8: Redact all Cell's with Header Author(s) as CBI_author"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $table: TableNode(hasHeader("Author"))
+    then
+        $table.streamTableCellsWithHeader("Author")
+            .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(true);
+                redactionEntity.addMatchedRule(8);
+                redactionEntity.addEngine(Engine.RULE);
+                redactionEntity.setRedactionReason("Author found");
+                redactionEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
+                insert(redactionEntity);
+            });
+    end
+
+rule "10: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+        $table: TableNode(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
+    then
+        $table.getEntitiesOfType("CBI_author").forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
+    end
+
+rule "14: Add CBI_author with \"et al.\" Regex"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    when
+         $section: SectionNode(containsString("et al."))
+    then
+         entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section)
+            .forEach(entity -> {
+                entity.setRedaction(true);
+                entity.setRedactionReason("Author found by \"et al\" regex");
+                entity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2g)");
+                entity.addMatchedRule(14);
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+                dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false);
+            });
+    end
+
+rule "16: Add recommendation for Addresses in Test Organism sections"
+    when
+        $section: SectionNode(excludesTables, containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.setRedactionReason("Line after \"Source\" in Test Organism Section");
+                entity.addEngine(Engine.RULE);
+                entity.addMatchedRule(16);
+                insert(entity);
+            });
+    end
+
+rule "17: Add recommendation for Addresses in Test Animals sections"
+    when
+        $section: SectionNode(excludesTables, containsString("Species:"), containsString("Source:"))
+    then
+        entityCreationService.lineAfterString("Source:",  "CBI_address", EntityType.RECOMMENDATION, $section)
+            .forEach(entity -> {
+                entity.setRedactionReason("Line after \"Source:\" in Test Animals Section");
+                entity.addEngine(Engine.RULE);
+                entity.addMatchedRule(17);
+                insert(entity);
+            });
+    end
+
+rule "18: Do not redact Names and Addresses if published information found in section without tables"
+    when
+        $section: SectionNode(excludesTables &&
+                              hasEntitiesOfType("published_information"),
+                             (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+        List<RedactionEntity>  publishedInformationEntities = $section.getEntitiesOfType("published_information");
+        $section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(false);
+                redactionEntity.setRedactionReason("Published Information found");
+                redactionEntity.addReferences(publishedInformationEntities);
+            });
+    end
+
+rule "18.0: Do not redact Names and Addresses if published information found in same table row"
+    when
+        $table: TableNode(hasEntitiesOfType("published_information"),
+                         (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
+    then
+
+        $table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("CBI_author", "CBI_address"))
+            .forEach(redactionEntity -> {
+                redactionEntity.setRedaction(false);
+                redactionEntity.setRedactionReason("Published Information found in row");
+                redactionEntity.addReferences($table.getEntitiesOfTypeInSameRow("published_information", redactionEntity));
+            });
+    end
+
+// --------------------------------------- PII rules -------------------------------------------------------------------
+
+rule "19: Redact all PII"
+    when
+        $pii: RedactionEntity(type == "PII", redaction == false)
+    then
+        $pii.setRedaction(true);
+        $pii.setRedactionReason("Personal Information found");
+        $pii.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+        $pii.addMatchedRule(19);
+    end
+
+rule "21: Redact Emails by RegEx (Non vertebrate study)"
+    when
+        $section: SectionNode(containsString("@"))
+    then
+        entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
+            .forEach(emailEntity -> {
+                emailEntity.setRedaction(true);
+                emailEntity.setRedactionReason("Found by Email Regex");
+                emailEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+                emailEntity.addMatchedRule(21);
+                insert(emailEntity);
+            });
+    end
+
+
+rule "25: Redact Phone and Fax by RegEx"
+    when
+        $section: SectionNode(containsString("Contact") ||
+                              containsString("Telephone") ||
+                              containsString("Phone") ||
+                              containsString("Ph.") ||
+                              containsString("Fax") ||
+                              containsString("Tel") ||
+                              containsString("Ter") ||
+                              containsString("Mobile") ||
+                              containsString("Fel") ||
+                              containsString("Fer"))
+    then
+        entityCreationService.byRegex("\\b(contact|telephone|phone|ph\\.|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, 2, $section)
+            .forEach(contactEntity -> {
+                contactEntity.setRedaction(true);
+                contactEntity.setRedactionReason("Found by Email Regex");
+                contactEntity.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+                contactEntity.addMatchedRule(26);
+                insert(contactEntity);
+            });
+    end
+
+
+rule "27: Redact AUTHOR(S)"
+    when
+        $section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("COMPLETION DATE:"), !containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(27);
+             authorEntity.setRedactionReason("AUTHOR(S) was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+
+rule "29: Redact AUTHOR(S)"
+    when
+        $section: SectionNode(excludesTables, containsString("AUTHOR(S):"), containsString("STUDY COMPLETION DATE:"))
+    then
+        entityCreationService.betweenStrings("AUTHOR(S):", "STUDY COMPLETION DATE:", "PII", EntityType.ENTITY, $section)
+         .forEach(authorEntity -> {
+             authorEntity.setRedaction(true);
+             authorEntity.addMatchedRule(29);
+             authorEntity.setRedactionReason("AUTHOR(S) was found");
+             authorEntity.setLegalBasis("Reg (EC) No 1107/2009 Art. 63 (2e)");
+             insert(authorEntity);
+         });
+    end
+
+rule "31: Redact PERFORMING LABORATORY (Non vertebrate study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
+    then
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+         .forEach(laboratoryEntity -> {
+             laboratoryEntity.setRedaction(false);
+             laboratoryEntity.addMatchedRule(31);
+             laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found for non vertebrate study");
+             dictionary.addLocalDictionaryEntry(laboratoryEntity);
+             insert(laboratoryEntity);
+         });
+    end
+
+rule "32: Redact PERFORMING LABORATORY (Vertebrate study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $section: SectionNode(excludesTables, containsString("PERFORMING LABORATORY:"))
+    then
+        entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
+         .forEach(laboratoryEntity -> {
+             laboratoryEntity.setRedaction(true);
+             laboratoryEntity.addMatchedRule(32);
+             laboratoryEntity.setRedactionReason("PERFORMING LABORATORY was found");
+             laboratoryEntity.setLegalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002");
+             dictionary.addLocalDictionaryEntry(laboratoryEntity);
+             insert(laboratoryEntity);
+         });
+    end
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "33: Purity Hint"
+    when
+        $section: SectionNode(containsStringIgnoreCase("purity"))
+    then
+        entityCreationService.byRegexIgnoreCase("(purity ?( of|\\(.{1,20}\\))?( ?:)?) .{0,5}[\\d\\.]+( .{0,4}\\.)? ?%", "hint_only", EntityType.ENTITY, 1, $section)
+            .forEach(hint -> {
+                hint.addEngine(Engine.RULE);
+                hint.addMatchedRule(33);
+            });
+    end
+
+rule "34: Redact signatures (not Vertebrate Study)"
+    when
+        not FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: ImageNode(imageType == ImageType.SIGNATURE)
+    then
+        $signature.setRedaction(true);
+        $signature.setMatchedRule(34);
+        $signature.setRedactionReason("Signature Found");
+        $signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+rule "35: Redact signatures (Vertebrate Study)"
+    when
+        FileAttribute(label == "Vertebrate Study", value == "Yes")
+        $signature: ImageNode(imageType == ImageType.SIGNATURE)
+    then
+        $signature.setRedaction(true);
+        $signature.setMatchedRule(35);
+        $signature.setRedactionReason("Signature Found");
+        $signature.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+
+rule "36: Redact logos"
+    when
+        $logo: ImageNode(imageType == ImageType.LOGO)
+    then
+        $logo.setRedaction(true);
+        $logo.setMatchedRule(36);
+        $logo.setRedactionReason("Logo Found");
+        $logo.setLegalBasis("Article 39(e)(3) of Regulation (EC) No 178/2002");
+    end
+
+// --------------------------------------- NER Entities rules -------------------------------------------------------------------
+
+rule "add NER Entities of type CBI_author"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
+    then
+        nerEntities.streamEntitiesOfType("CBI_author")
+            .map(nerEntity -> entityCreationService.byNerEntity(nerEntity, EntityType.RECOMMENDATION, document))
+            .forEach(entity -> insert(entity));
+    end
+
+rule "combine and add NER Entities as CBI_address"
+    salience 999
+    when
+        nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
+    then
+        nerEntitiesAdapter.combineNerEntitiesToCbiAddressDefaults(nerEntities)
+            .map(boundary -> entityCreationService.byBoundary(boundary, "CBI_address", EntityType.RECOMMENDATION, document))
+            .forEach(entity -> {
+                entity.addEngine(Engine.NER);
+                insert(entity);
+            });
+    end
+
+// --------------------------------------- manual redaction rules -------------------------------------------------------------------
+
+rule "Apply manual resize redaction"
+    salience 128
+    when
+        $resizeRedaction: ManualResizeRedaction($id: annotationId)
+        $entityToBeResized: RedactionEntity(matchesAnnotationId($id))
+    then
+        manualRedactionApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
+        retract($resizeRedaction);
+        update($entityToBeResized);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Entity"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToBeRemoved.removeFromGraph();
+        retract($entityToBeRemoved);
+    end
+
+rule "Apply id removals that are valid and not in forced redactions to Image"
+    salience 128
+    when
+        IdRemoval(status == AnnotationStatus.APPROVED, !removeFromDictionary, requestDate != null, $id: annotationId)
+        not ManualForceRedaction($id == annotationId, status == AnnotationStatus.APPROVED, requestDate != null)
+        $entityToBeRemoved: ImageNode($id == id)
+    then
+        $entityToBeRemoved.setIgnored(true);
+    end
+
+rule "Apply force redaction"
+    salience 128
+    when
+        ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED, requestDate != null, $legalBasis: legalBasis)
+        $entityToForce: RedactionEntity(matchesAnnotationId($id))
+    then
+        $entityToForce.setLegalBasis($legalBasis);
+        $entityToForce.setRedaction(true);
+        $entityToForce.setSkipRemoveEntitiesContainedInLarger(true);
+    end
+
+rule "Apply image recategorization"
+    salience 128
+    when
+        ManualImageRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $imageType: type)
+        $image: ImageNode($id == id)
+    then
+        $image.setImageType(parseImageType($imageType));
+    end
+
+// --------------------------------------- merging rules -------------------------------------------------------------------
+
+rule "remove Entity contained by Entity of same type"
+    salience 65
+    when
+        $larger: RedactionEntity($type: type, $entityType: entityType)
+        $contained: RedactionEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $contained.removeFromGraph();
+        retract($contained);
+    end
+
+rule "merge intersecting Entities of same type"
+    salience 64
+    when
+        $first: RedactionEntity($type: type, $entityType: entityType, !resized, !skipRemoveEntitiesContainedInLarger)
+        $second: RedactionEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $first.removeFromGraph();
+        $second.removeFromGraph();
+        RedactionEntity mergedEntity = entityCreationService.byEntities(List.of($first, $second), $type, $entityType, document);
+        retract($first);
+        retract($second);
+        insert(mergedEntity);
+    end
+
+rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
+    salience 64
+    when
+        $falsePositive: RedactionEntity($type: type, entityType == EntityType.FALSE_POSITIVE)
+        $entity: RedactionEntity(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.removeFromGraph();
+        retract($entity)
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
+    salience 64
+    when
+        $falseRecommendation: RedactionEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION)
+        $recommendation: RedactionEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
+    salience 256
+    when
+        $entity: RedactionEntity($type: type, entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $entity.addEngines($recommendation.getEngines());
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
+    salience 256
+    when
+        $entity: RedactionEntity(entityType == EntityType.ENTITY)
+        $recommendation: RedactionEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
+    then
+        $recommendation.removeFromGraph();
+        retract($recommendation);
+    end
+
+rule "remove Entity of lower rank, when equal boundaries and entityType"
+    salience 32
+    when
+        $higherRank: RedactionEntity($type: type, $entityType: entityType, $boundary: boundary)
+        $lowerRank: RedactionEntity($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !redaction)
+    then
+        $lowerRank.removeFromGraph();
+        retract($lowerRank);
+    end
+
+// --------------------------------------- FileAttribute Rules -------------------------------------------------------------------
+
+rule "remove duplicate FileAttributes"
+    salience 64
+    when
+        $fileAttribute: FileAttribute($label: label, $value: value)
+        $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
+    then
+        retract($duplicate);
+    end
+
+// --------------------------------------- local dictionary search -------------------------------------------------------------------
+
+rule "run local dictionary search"
+    agenda-group "LOCAL_DICTIONARY_ADDS"
+    salience -999
+    when
+        DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
+    then
+        entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document)
+            .forEach(entity -> {
+                entity.addEngine(Engine.RULE);
+                insert(entity);
+            });
+    end