Pull request #21: Cleaned up dictionaries

Merge in RED/redaction-service from cleanDictionaries to master

* commit 'b5412dc9590d15dcae9f87f7fc9b7ea50e4c63ae':
  Cleaned up dictionaries

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -39,7 +39,7 @@
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.6</version>
+            <version>1.0.12</version>
         </dependency>
         <dependency>
             <groupId>org.drools</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/classification/model/Document.java

@@ -4,7 +4,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 import com.iqser.red.service.redaction.v1.model.RedactionLogEntry;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
@@ -18,7 +17,7 @@ public class Document {
 
     private List<Page> pages = new ArrayList<>();
     private List<Paragraph> paragraphs = new ArrayList<>();
-    private Map<Integer, Set<Entity>> entities = new HashMap<>();
+    private Map<Integer, List<Entity>> entities = new HashMap<>();
     private FloatFrequencyCounter textHeightCounter = new FloatFrequencyCounter();
     private FloatFrequencyCounter fontSizeCounter= new FloatFrequencyCounter();
     private StringFrequencyCounter fontCounter= new StringFrequencyCounter();

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/parsing/model/TextPositionSequence.java

@@ -39,6 +39,12 @@ public class TextPositionSequence implements CharSequence {
         return text.charAt(0);
     }
 
+    public char charAt(int index, boolean caseInSensitive) {
+        TextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
+    }
+
     @Override
     public TextPositionSequence subSequence(int start, int end) {
         return new TextPositionSequence(textPositions.subList(start, end), page);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Entity.java

@@ -1,14 +1,16 @@
 package com.iqser.red.service.redaction.v1.server.redaction.model;
 
-
 import java.util.ArrayList;
 import java.util.List;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 
 @Data
+@EqualsAndHashCode(onlyExplicitlyIncluded = true)
 public class Entity {
 
+    @EqualsAndHashCode.Include
     private final String word;
     private final String type;
     private boolean redaction;
@@ -16,10 +18,17 @@ public class Entity {
     private List<EntityPositionSequence> positionSequences = new ArrayList<>();
     private Integer start;
     private Integer end;
+
+    @EqualsAndHashCode.Include
     private String headline;
     private int matchedRule;
 
-    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule) {
+    @EqualsAndHashCode.Include
+    private int sectionNumber;
+
+
+    public Entity(String word, String type, boolean redaction, String redactionReason, List<EntityPositionSequence> positionSequences, String headline, int matchedRule, int sectionNumber) {
+
         this.word = word;
         this.type = type;
         this.redaction = redaction;
@@ -27,13 +36,18 @@ public class Entity {
         this.positionSequences = positionSequences;
         this.headline = headline;
         this.matchedRule = matchedRule;
+        this.sectionNumber = sectionNumber;
     }
 
-    public Entity(String word, String type, Integer start, Integer end, String headline) {
+
+    public Entity(String word, String type, Integer start, Integer end, String headline, int sectionNumber) {
+
         this.word = word;
         this.type = type;
         this.start = start;
         this.end = end;
         this.headline = headline;
+        this.sectionNumber = sectionNumber;
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/EntityPositionSequence.java

@@ -6,15 +6,20 @@ import java.util.UUID;
 
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 
+import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 
-
 @Data
 @RequiredArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode
 public class EntityPositionSequence {
 
+    @EqualsAndHashCode.Exclude
     private List<TextPositionSequence> sequences = new ArrayList<>();
     private int pageNumber;
     private final UUID id;
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -13,45 +13,65 @@ public class SearchableText {
 
     private List<TextPositionSequence> sequences = new ArrayList<>();
 
+
     public void add(TextPositionSequence textPositionSequence) {
+
         sequences.add(textPositionSequence);
     }
 
+
     public void addAll(List<TextPositionSequence> textPositionSequences) {
+
         sequences.addAll(textPositionSequences);
     }
 
 
-    public List<EntityPositionSequence> getSequences(String searchString) {
+    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {
 
-        char[] searchChars = searchString.replaceAll("\\n", " ").toCharArray();
+        String normalizedSearchString;
+        if (caseInsensitive) {
+            normalizedSearchString = searchString.toLowerCase();
+        } else {
+            normalizedSearchString = searchString;
+        }
+
+        char[] searchChars = normalizedSearchString.replaceAll("\\n", " ").toCharArray();
         int counter = 0;
 
-
         List<TextPositionSequence> crossSequenceParts = new ArrayList<>();
         List<EntityPositionSequence> finalMatches = new ArrayList<>();
         for (int i = 0; i < sequences.size(); i++) {
             TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
             for (int j = 0; j < sequences.get(i).length(); j++) {
 
-                if(i > 0 && j == 0 && sequences.get(i).charAt(0) == ' ' && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) == ' '
-                || j > 0 && sequences.get(i).charAt(j) == ' ' && sequences.get(i).charAt(j - 1) == ' '){
-                    if(j == sequences.get(i).length() -1  && counter != 0 && !partMatch.getTextPositions().isEmpty()){
+                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInsensitive) == ' ' && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == sequences.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
                         crossSequenceParts.add(partMatch);
                     }
                     continue;
                 }
 
-                if(j == 0 && sequences.get(i).charAt(j) != ' ' && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && searchChars[counter] == ' '){
+                if (j == 0 && sequences.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1)
+                                .length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
                     counter++;
                 }
 
-                if (sequences.get(i).charAt(j) == searchChars[counter] || counter != 0 && sequences.get(i).charAt(j) == '-') {
+                if (sequences.get(i)
+                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == '-') {
 
-                    if(counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i).charAt(j - 1)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1))
-                            || j == 0 && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && sequences.get(i).charAt(j) != ' ') {
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1)
+                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) != ' ' && sequences.get(i)
+                            .charAt(j, caseInsensitive) != ' ') {
                         partMatch.add(sequences.get(i).textPositionAt(j));
-                        if (!(j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) == '-' && searchChars[counter] != '-')) {
+                        if (!(j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                             counter++;
                         }
                     }
@@ -59,10 +79,13 @@ public class SearchableText {
                     if (counter == searchString.length()) {
                         crossSequenceParts.add(partMatch);
 
-                        if(i == sequences.size() - 1 && j == sequences.get(i).length() -1
-                                || j != sequences.get(i).length() -1 && isSeparator(sequences.get(i).charAt(j +1))
-                                || j == sequences.get(i).length() -1 && isSeparator(sequences.get(i + 1).charAt(0))
-                                || j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) != ' ' && sequences.get(i + 1).charAt(0) != ' ') {
+                        if (i == sequences.size() - 1 && j == sequences.get(i).length() - 1 || j != sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i)
+                                .charAt(j + 1, caseInsensitive)) || j == sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && sequences.get(i + 1)
+                                .charAt(0, caseInsensitive) != ' ') {
                             finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
                         }
 
@@ -72,14 +95,14 @@ public class SearchableText {
                     }
                 } else {
                     counter = 0;
-                    if(!crossSequenceParts.isEmpty()){
+                    if (!crossSequenceParts.isEmpty()) {
                         j--;
                     }
                     crossSequenceParts = new ArrayList<>();
                     partMatch = new TextPositionSequence(sequences.get(i).getPage());
                 }
 
-                if(j == sequences.get(i).length() -1  && counter != 0){
+                if (j == sequences.get(i).length() - 1 && counter != 0) {
                     crossSequenceParts.add(partMatch);
                 }
             }
@@ -89,18 +112,18 @@ public class SearchableText {
     }
 
 
-    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts){
+    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {
 
         UUID id = UUID.randomUUID();
         List<EntityPositionSequence> result = new ArrayList<>();
         int currentPage = -1;
         EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
-        for (TextPositionSequence textPositionSequence :crossSequenceParts){
-            if(currentPage == -1){
+        for (TextPositionSequence textPositionSequence : crossSequenceParts) {
+            if (currentPage == -1) {
                 currentPage = textPositionSequence.getPage();
                 entityPositionSequence.setPageNumber(currentPage);
                 entityPositionSequence.getSequences().add(textPositionSequence);
-            } else if(currentPage == textPositionSequence.getPage()){
+            } else if (currentPage == textPositionSequence.getPage()) {
                 entityPositionSequence.getSequences().add(textPositionSequence);
             } else {
                 result.add(entityPositionSequence);
@@ -114,13 +137,14 @@ public class SearchableText {
 
 
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
 
-
     @Override
     public String toString() {
+
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;
@@ -137,10 +161,14 @@ public class SearchableText {
             previous = word;
         }
 
-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll("  ", " ");
+        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
+                .replaceAll("\n", " ")
+                .replaceAll("  ", " ");
     }
 
-    public String getAsStringWithLinebreaks(){
+
+    public String getAsStringWithLinebreaks() {
+
         StringBuilder sb = new StringBuilder();
 
         TextPositionSequence previous = null;

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -25,6 +25,8 @@ public class Section {
 
     private String headline;
 
+    private int sectionNumber;
+
 
     public boolean contains(String type) {
 
@@ -32,6 +34,12 @@ public class Section {
     }
 
 
+    public boolean headlineContainsWord(String word) {
+
+        return StringUtils.containsIgnoreCase(headline, word);
+    }
+
+
     public void redact(String type, int ruleNumber, String reason) {
 
         entities.forEach(entity -> {
@@ -58,11 +66,15 @@ public class Section {
 
     public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {
 
-        String value = StringUtils.substringBetween(text, start, "\n");
+        String[] values = StringUtils.substringsBetween(text, start, "\n");
 
-        if (value != null) {
-            Set<Entity> found = findEntity(value.trim(), asType);
-            entities.addAll(found);
+        if (values != null) {
+            for (String value : values) {
+                if (StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntity(value.trim(), asType);
+                    entities.addAll(found);
+                }
+            }
         }
 
         // TODO No need to iterate
@@ -79,11 +91,15 @@ public class Section {
 
     public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason) {
 
-        String value = StringUtils.substringBetween(searchText, start, stop);
+        String[] values = StringUtils.substringsBetween(searchText, start, stop);
 
-        if (value != null) {
-            Set<Entity> found = findEntity(value.trim(), asType);
-            entities.addAll(found);
+        if (values != null) {
+            for (String value : values) {
+                if (value != null && StringUtils.isNotBlank(value)) {
+                    Set<Entity> found = findEntity(value.trim(), asType);
+                    entities.addAll(found);
+                }
+            }
         }
 
         // TODO No need to iterate
@@ -109,13 +125,11 @@ public class Section {
 
             if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText
                     .charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
-                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
             }
         } while (startIndex > -1);
 
-        removeEntitiesContainedInLarger(found);
-
-        return found;
+        return removeEntitiesContainedInLarger(found);
     }
 
 
@@ -125,7 +139,7 @@ public class Section {
     }
 
 
-    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+    public Set<Entity> removeEntitiesContainedInLarger(Set<Entity> entities) {
 
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
@@ -137,6 +151,7 @@ public class Section {
             }
         }
         entities.removeAll(wordsToRemove);
+        return entities;
     }
 
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -41,6 +41,9 @@ public class DictionaryService {
     @Getter
     private List<String> caseInsensitiveTypes = new ArrayList<>();
 
+    @Getter
+    private float[] defaultColor;
+
 
     public void updateDictionary() {
 
@@ -60,10 +63,6 @@ public class DictionaryService {
                 entryColors = typeResponse.getTypes()
                         .stream()
                         .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
-                dictionary = entryColors.keySet()
-                        .stream()
-                        .collect(Collectors.toMap(type -> type, s -> new HashSet<>(dictionaryClient.getDictionaryForType(s)
-                                .getEntries())));
                 hintTypes = typeResponse.getTypes()
                         .stream()
                         .filter(TypeResult::isHint)
@@ -74,6 +73,8 @@ public class DictionaryService {
                         .filter(TypeResult::isCaseInsensitive)
                         .map(TypeResult::getType)
                         .collect(Collectors.toList());
+                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, this::convertEntries));
+                defaultColor = dictionaryClient.getDefaultColor().getColor();
             }
         } catch (FeignException e) {
             log.warn("Got some unknown feignException", e);
@@ -81,4 +82,16 @@ public class DictionaryService {
         }
     }
 
+
+    private Set<String> convertEntries(String s) {
+        if (caseInsensitiveTypes.contains(s)) {
+            return dictionaryClient.getDictionaryForType(s)
+                    .getEntries()
+                    .stream()
+                    .map(String::toLowerCase)
+                    .collect(Collectors.toSet());
+        }
+        return new HashSet<>(dictionaryClient.getDictionaryForType(s).getEntries());
+    }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -1,6 +1,7 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -13,6 +14,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.Document;
 import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
 import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.SearchableText;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Section;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
@@ -27,19 +29,20 @@ public class EntityRedactionService {
     private final DictionaryService dictionaryService;
     private final DroolsExecutionService droolsExecutionService;
 
+
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
         droolsExecutionService.updateRules();
 
         Set<Entity> documentEntities = new HashSet<>();
+        int sectionNumber = 1;
         for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
 
             SearchableText searchableText = paragraph.getSearchableText();
 
             List<Table> tables = paragraph.getTables();
 
-            List<SearchableText> searchableRows = new ArrayList<>();
             for (Table table : tables) {
                 for (List<Cell> row : table.getRows()) {
                     SearchableText searchableRow = new SearchableText();
@@ -51,92 +54,129 @@ public class EntityRedactionService {
                             searchableRow.addAll(textBlock.getSequences());
                         }
                     }
-                    searchableRows.add(searchableRow);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline(), sectionNumber);
+
+                    Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
+                            .entities(rowEntities)
+                            .text(searchableRow.getAsStringWithLinebreaks())
+                            .searchText(searchableRow.toString())
+                            .headline(table.getHeadline())
+                            .sectionNumber(sectionNumber)
+                            .build());
+
+                    documentEntities.addAll(clearAndFindPositions(analysedRowSection.getEntities(), searchableRow));
+                    sectionNumber++;
                 }
+                sectionNumber++;
             }
 
-            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline());
-            Section analysedSection = droolsExecutionService.executeRules(Section
-                    .builder()
+            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline(), sectionNumber);
+            Section analysedSection = droolsExecutionService.executeRules(Section.builder()
                     .entities(entities)
                     .text(searchableText.getAsStringWithLinebreaks())
                     .searchText(searchableText.toString())
                     .headline(paragraph.getHeadline())
+                    .sectionNumber(sectionNumber)
                     .build());
 
-            for (Entity entity : analysedSection.getEntities()) {
-                entity.setPositionSequences(searchableText.getSequences(entity.getWord()));
+            documentEntities.addAll(clearAndFindPositions(analysedSection.getEntities(), searchableText));
+            sectionNumber++;
+        }
+
+        for (Entity entity : documentEntities) {
+            Map<Integer, List<EntityPositionSequence>> sequenceOnPage = new HashMap<>();
+            for (EntityPositionSequence entityPositionSequence : entity.getPositionSequences()) {
+                sequenceOnPage.computeIfAbsent(entityPositionSequence.getPageNumber(), (x) -> new ArrayList<>())
+                        .add(entityPositionSequence);
             }
 
-            documentEntities.addAll(analysedSection.getEntities());
-
-            for (SearchableText searchableRow : searchableRows) {
-                Set<Entity> rowEntities = findEntities(searchableRow, "//TODO TableHeader");
-
-                Section analysedRowSection = droolsExecutionService.executeRules(Section
-                        .builder()
-                        .entities(rowEntities)
-                        .text(searchableRow.getAsStringWithLinebreaks())
-                        .searchText(searchableRow.toString())
-                        .headline("//TODO TableHeader")
-                        .build());
-
-                for (Entity entity : analysedRowSection.getEntities()) {
-                    entity.setPositionSequences(searchableRow.getSequences(entity.getWord()));
-                }
-                documentEntities.addAll(analysedRowSection.getEntities());
+            for (Map.Entry<Integer, List<EntityPositionSequence>> entry : sequenceOnPage.entrySet()) {
+                classifiedDoc.getEntities()
+                        .computeIfAbsent(entry.getKey(), (x) -> new ArrayList<>())
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), entry
+                                .getValue(), entity.getHeadline(), entity.getMatchedRule(), entity.getSectionNumber()));
             }
         }
 
-        documentEntities.forEach(entity -> {
-            entity.getPositionSequences().forEach(sequence -> {
-                classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
-                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence), entity.getHeadline(), entity.getMatchedRule())
-                );
-            });
-        });
     }
 
-    private Set<Entity> findEntities(SearchableText searchableText, String headline) {
 
-        String normalizedInputString = searchableText.toString();
+    private Set<Entity> clearAndFindPositions(Set<Entity> entities, SearchableText text) {
+
+        removeEntitiesContainedInLarger(entities);
+
+        for (Entity entity : entities) {
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                entity.setPositionSequences(text.getSequences(entity.getWord(), true));
+            } else {
+                entity.setPositionSequences(text.getSequences(entity.getWord(), false));
+            }
+        }
+
+        return entities;
+    }
+
+
+    private Set<Entity> findEntities(SearchableText searchableText, String headline, int sectionNumber) {
+
+        String inputString = searchableText.toString();
+        String lowercaseInputString = inputString.toLowerCase();
 
         Set<Entity> found = new HashSet<>();
         for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
-            for (String value : entry.getValue()) {
-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = normalizedInputString.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();
 
-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(normalizedInputString.charAt(startIndex - 1)) || isSeparator(normalizedInputString.charAt(startIndex - 1))) &&
-                            (stopIndex == normalizedInputString.length() || isSeparator(normalizedInputString.charAt(stopIndex)))) {
-                        found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex, headline));
-                    }
-                } while (startIndex > -1);
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())) {
+                found.addAll(find(lowercaseInputString, entry.getValue(), entry.getKey(), headline, sectionNumber));
+            } else {
+                found.addAll(find(inputString, entry.getValue(), entry.getKey(), headline, sectionNumber));
             }
         }
 
         removeEntitiesContainedInLarger(found);
 
         return found;
+
     }
 
+
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline, int sectionNumber) {
+
+        Set<Entity> found = new HashSet<>();
+        for (String value : values) {
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(value, stopIndex);
+                stopIndex = startIndex + value.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline, sectionNumber));
+                }
+            } while (startIndex > -1);
+        }
+        return found;
+    }
+
+
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
+
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                     wordsToRemove.add(inner);
                 }
             }
         }
         entities.removeAll(wordsToRemove);
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -17,7 +17,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 @SuppressWarnings("all")
 public class SectionsBuilderService {
 
-
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
@@ -29,7 +28,8 @@ public class SectionsBuilderService {
         for (Page page : document.getPages()) {
             for (AbstractTextContainer current : page.getTextBlocks()) {
 
-                if (current.getClassification() == null || current.getClassification().equals("Header") || current.getClassification().equals("Footer")) {
+                if (current.getClassification() == null || current.getClassification()
+                        .equals("Header") || current.getClassification().equals("Footer")) {
                     continue;
                 }
 
@@ -37,7 +37,7 @@ public class SectionsBuilderService {
 
                 if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
 
-                    Paragraph chunkBlock = buildTextBlock(chunkWords);
+                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     lastHeadline = current.getText();
                     chunkBlockList.add(chunkBlock);
@@ -51,7 +51,7 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords);
+        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         if (chunkBlock != null) {
             chunkBlockList.add(chunkBlock);
             chunkBlock.setHeadline(lastHeadline);
@@ -61,7 +61,7 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList) {
+    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
         Paragraph paragraph = new Paragraph();
         TextBlock textBlock = null;
@@ -70,17 +70,23 @@ public class SectionsBuilderService {
         boolean splitByTable = false;
 
         Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
-        boolean alreadyAdded= false;
+        boolean alreadyAdded = false;
+        AbstractTextContainer previous = null;
         while (itty.hasNext()) {
             AbstractTextContainer container = itty.next();
 
             if (container instanceof Table) {
                 splitByTable = true;
 
+                if (previous != null && previous instanceof TextBlock && previous.getText().startsWith("Table ")) {
+                    ((Table) container).setHeadline(previous.getText());
+                } else {
+                    ((Table) container).setHeadline("Table in: " + lastHeadline);
+                }
 
                 if (textBlock != null && !alreadyAdded) {
                     paragraph.getPageBlocks().add(textBlock);
-                    alreadyAdded =true;
+                    alreadyAdded = true;
                 }
                 paragraph.getPageBlocks().add(container);
                 continue;
@@ -89,24 +95,28 @@ public class SectionsBuilderService {
             TextBlock wordBlock = (TextBlock) container;
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
                 alreadyAdded = false;
             } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                 textBlock.setPage(pageBefore);
                 paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
-                        spatialEntity.getWidth(), spatialEntity.getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                        .getHeight());
             }
             pageBefore = wordBlock.getPage();
             splitByTable = false;
+            previous = container;
         }
 
         if (textBlock != null && !alreadyAdded) {
@@ -115,5 +125,4 @@ public class SectionsBuilderService {
         return paragraph;
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -13,6 +13,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
 import lombok.Getter;
+import lombok.Setter;
 
 @SuppressWarnings("all")
 public class Table extends AbstractTextContainer {
@@ -21,6 +22,10 @@ public class Table extends AbstractTextContainer {
 
     private RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
 
+    @Getter
+    @Setter
+    private String headline;
+
     @Getter
     private int rowCount = 0;
     @Getter

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -203,7 +203,7 @@ public class AnnotationHighlightService {
         }
 
         if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
-            return dictionaryService.getEntryColors().get("default");
+            return dictionaryService.getDefaultColor();
         }
 
         return dictionaryService.getEntryColors().get(entity.getType());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -35,6 +35,7 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.iqser.red.service.configuration.v1.api.model.DefaultColor;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -57,7 +58,9 @@ public class RedactionIntegrationTest {
     private static final String ADDRESS_CODE = "address";
     private static final String NAME_CODE = "name";
     private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String DEFAULT = "default";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
+    private static final String MUST_REDACT = "must_redact";
 
     @Autowired
     private RedactionController redactionController;
@@ -71,6 +74,7 @@ public class RedactionIntegrationTest {
     private final Map<String, List<String>> dictionary = new HashMap<>();
     private final Map<String, float[]> typeColorMap = new HashMap<>();
     private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();
 
     @TestConfiguration
     public static class RedactionIntegrationTestConfiguration {
@@ -82,7 +86,8 @@ public class RedactionIntegrationTest {
 
             KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
             InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
-            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
             KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
             kieBuilder.buildAll();
             KieModule kieModule = kieBuilder.getKieModule();
@@ -107,7 +112,10 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
         when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
         when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
-        when(dictionaryClient.getDictionaryForType(DEFAULT)).thenReturn(getDictionaryResponse(DEFAULT));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
+        when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
+        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
     }
 
 
@@ -129,11 +137,25 @@ public class RedactionIntegrationTest {
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt")
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(MUST_REDACT, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/must_redact.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
-        dictionary.put(DEFAULT, new ArrayList<>());
     }
 
 
@@ -148,14 +170,26 @@ public class RedactionIntegrationTest {
         typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
         typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
         typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
-        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
-        typeColorMap.put(DEFAULT, new float[]{1, 0.502f, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f});
+        typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f});
+        typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f});
+        typeColorMap.put(MUST_REDACT, new float[]{1, 0, 0});
 
         hintTypeMap.put(VERTEBRATES_CODE, true);
         hintTypeMap.put(ADDRESS_CODE, false);
         hintTypeMap.put(NAME_CODE, false);
         hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(DEFAULT, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
+        hintTypeMap.put(MUST_REDACT, true);
+
+        caseInSensitiveMap.put(VERTEBRATES_CODE, true);
+        caseInSensitiveMap.put(ADDRESS_CODE, false);
+        caseInSensitiveMap.put(NAME_CODE, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
+        caseInSensitiveMap.put(MUST_REDACT, true);
     }
 
 
@@ -166,14 +200,22 @@ public class RedactionIntegrationTest {
                 .map(typeColor -> TypeResult.builder()
                         .type(typeColor.getKey())
                         .color(typeColor.getValue())
-                        .isHint(hintTypeMap.get(typeColor.getKey())).build())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .build())
+
                 .collect(Collectors.toList());
     }
 
 
     private DictionaryResponse getDictionaryResponse(String type) {
 
-        return DictionaryResponse.builder().color(typeColorMap.get(type)).entries(dictionary.get(type)).isHint(hintTypeMap.get(type)).build();
+        return DictionaryResponse.builder()
+                .color(typeColorMap.get(type))
+                .entries(dictionary.get(type))
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .build();
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -0,0 +1,50 @@
+package com.iqser.red.service.redaction.v1.server.redaction.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.kie.api.runtime.KieContainer;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest
+public class EntityRedactionServiceTest {
+
+    @MockBean
+    private KieContainer kieContainer;
+
+    @MockBean
+    private DroolsExecutionService droolsExecutionService;
+
+    @MockBean
+    private DictionaryService dictionaryService;
+
+    @Autowired
+    private EntityRedactionService entityRedactionService;
+
+
+    @Test
+    public void testNestedEntitiesRemoval() {
+
+        Set<Entity> entities = new HashSet<>();
+        Entity nested = new Entity("nested", "fake type", 10, 16, "fake headline", 0);
+        Entity nesting = new Entity("nesting nested", "fake type", 2, 16, "fake headline", 0);
+        entities.add(nested);
+        entities.add(nesting);
+        entityRedactionService.removeEntitiesContainedInLarger(entities);
+
+        assertThat(entities.size()).isEqualTo(1);
+        assertThat(entities).contains(nesting);
+
+    }
+
+}

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt

@@ -1,3 +0,0 @@
-In Vitro
-In vitro
-in vitro

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt

@@ -0,0 +1,2 @@
+guideline
+unpublished

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt

@@ -0,0 +1,3 @@
+Batches Produced at
+CTL
+for determination of residues

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt

@@ -0,0 +1,3 @@
+published paper
+in vitro
+in-vitro

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt

@@ -0,0 +1,9 @@
+in vivo
+in-vivo
+dermal penetration
+oral toxicity
+oral-toxicity
+acute toxicity
+acute-toxicity
+eco toxicity
+eco-toxicity

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt

@@ -1,48 +1,63 @@
-Vulpes vulpes
+a. sylvaticus
 african clawed frog
 agalychnis callidryas
+albino rat
+american bullfrog tadpole
+american toad
 amphibian
 amphibians
-American bullfrog tadpole
-american toad
 anad platyrhynchos
-Anas platyrhynchos
+anas platyrhynchos
 anuran
 anurans
+apodemus
+apodemus flavicollis
 apodemus syl vaticus
+apodemus sylvaticus
+arvicola terrestris
 avian
+bank vole
 bird
 birds
 bluegill
 bluegill sunfish
 bobwhite
 bobwhite quail
-bullfrog
-Bufo americanus
 brachydanio rerio
+brown hare
+bufo americanus
+bullfrog
 canary
 carassius carassius
 carp
+catesbeiana
 catfish
 cattle
 cattles
 channel catfish
-Chinook
 chicken
-Colinus virginianus
+chinese hamster
+chinese hamsters
+chinook
+coho salmon
 colinus virginianus
-Common carp
+columba palumbus
+columbidae
+common carp
+common vole
 coturnix japonica
-Coturnix japonica
 cow
 cows
-Crucian carp
+crocidura russula
+crucian carp
 cyprinodon variegatus
 cyprinus carpio
 dog
 dogs
 duck
 ducks
+european brown hare
+european rabbit
 fathead minnow
 fish
 fishes
@@ -56,56 +71,84 @@ galaxias truttaceus
 gasterosteus aculeatus
 goat
 goats
+greater white-toothed shrew
 guinea
 guinea pig
 guinea pigs
-Guppy
+guinea-pigs
+guppy
 hamster
 hamsters
 hen
 hens
-Hyla versicolor
+house mouse
+hyla versicolor
 ictalurus melas
 ictalurus punctatus
 japanese quail
 japonica
+kisutch
+lagomorph
 lebistes reticulatus
 leiostomus xanthurus
 leisostomus xanthurus
 lepomis macrochirus
+lepus europaeus
+limnocharis
+limnodynastes
+limnodynastes tasmaniensis
 livestock
 livestocks
+mallard
 mallard duck
 mammal
+mammalian
 mammals
-Mammalian
+marten
+martes
 mice
+microtus
+microtus agrestis
+microtus arvalis
+microtus subterraneus
 midwestern anurans
+minnow
+minnows
 monkey
 mouse
+mus musculus
+myodes glareolus
 northern bobwhite
+o. cuniculus
 o. mykiss
-Oncorhynchus mykiss
-Oncorhynchus
-O. mykiss
+o. tshawytscha
+oncorhynchus
+oncorhynchus mykiss
+oncorhynchus tshawytscha
+oryctolagus cuniculus
 oryzias melastigma
 oryzias melastigma larvae
 p. promelas
 pagrus major
+palumbus
 pig
 pigeon
 pigeons
+pigs
+pimephales promela
 pimephales promelas
-Pseudacris triseriata
 poecilia reticulata
 poultry
+pseudacris
+pseudacris triseriata
 quail
+r. catesbeiana
 rabbit
 rabbits
 rainbow trout
-Rana limnocharis
 rana
-limnocharis
+rana catesbeiana
+rana limnocharis
 rana pipiens
 rat
 rats
@@ -114,120 +157,35 @@ reptiles
 ricefish
 ruminant
 ruminants
+salmo gairdneri
+salmon
+serinus canaria
 sheepshead minnow
 sheepshead minnows
-spea multiplicata
-Salmo gairdneri
-salmon
-spotted march frog
-tadpoles
-treefrog
-toad
-terrestrial vertrebrates
-Limnodynastes tasmaniensis
-trout
-Vulpes vulpes
-wistar
-xenopus laevis
-xenpous leavis
-zebra fish
-zebrafish
-Salmo gairdneri
-minnow
-minnows
-Pimephales promela
-Cyprinodon variegatus
-limnodynastes
-Rana catesbeiana
-R. catesbeiana
-coho salmon
-Oncorhynchus tshawytscha
-O. tshawytscha
-tshawytscha
-catesbeiana
-kisutch
-Pseudacris triseriata
-Pseudacris
-triseriata
-Wood pigeon
-Columba palumbus
-palumbus
-Columbidae
 shrew
 shrews
-bank vole
-common vole
+sorex araneus
+spea multiplicata
+spotted march frog
+tadpoles
+terrestrial vertrebrates
+toad
+treefrog
+triseriata
+trout
+tshawytscha
 vole
 voles
-lagomorph
-Wood mouse
-Apodemus sylvaticus
-A. sylvaticus
-Apodemus flavicollis
-Apodemus
-mus musculus
-Microtus arvalis
-Microtus agrestis
-Microtus
-Arvicola terrestris
-Sorex araneus
-Myodes glareolus
-yellow-necked mouse
-house mouse
-Oryctolagus cuniculus
-marten
-martes
+vulpes vulpes
+white rabbits
 white-toothed shrew
-greater white-toothed shrew
-Lepus europaeus
-brown hare
-European brown hare
-European rabbit
-O. cuniculus
-Crocidura russula
-Chinese Hamster
-Rat
-Rats
-Dog
-Chinese hamsters
-Chinese hamster
-Mouse
-Guinea pig
-Wistar rats
-Rabbit
-mammalian
-Japanese quail
-Microtus subterraneus
-Lepomis macrochirus
-P. promelas
-Cyprinus carpio
-Fish
-Ictalurus punctatus
-Carassius carassius
-Lepomis macrochirus
-Poecilia reticulata
-Lebistes reticulatus
-Lepomis macrochirus
-Leiostomus xanthurus
-Pimephales promelas
-Lepomis macrochirus
-Albino rat
-Hen
-Goat
-Livestock
-Guinea Pigs
-Hamster
+wistar
+wistar rats
+wood mice
 wood mouse
-Rabbits
-Mice
-Rainbow trout
-Canary
-Serinus canaria
-Guinea Pig
-Cow
-Pigs
-Poultry
-Guinea-pigs
-White rabbits
-Birds
-Wood mice
+wood pigeon
+xenopus laevis
+xenpous leavis
+yellow-necked mouse
+zebra fish
+zebrafish

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -32,27 +32,70 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
 	end
 
 
-rule "4: Redact contact information, if applicant is found"
+rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
 	when
-		eval(section.getText().toLowerCase().contains("applicant"));
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true);
 	then
-		section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4");
-		section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Contact point:",  "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4");
+		section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
+		section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
 	end
 
 
-rule "5: Redact contact information, if 'Producer of the plant protection product' is found"
+rule "5: Do not redact in guideline sections"
 	when
-		eval(section.getText().contains("Producer of the plant protection product"));
+		eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance"));
 	then
-		section.redactLineAfter("Name:", "address", 5, "xxxx");
-		section.redactBetween("Address:", "Contact", "address", 5, "xxxx");
-		section.redactBetween("Contact:", "Phone", "address", 5, "xxxx");
-		section.redactLineAfter("Phone:", "address", 5, "xxxx");
-		section.redactLineAfter("Fax:", "address", 5, "xxxx");
-		section.redactLineAfter("E-mail:", "address", 5, "xxxx");
-	end
+		section.redactNot("name", 5, "Section is a guideline section.");
+		section.redactNot("address", 5, "Section is a guideline section.");
+	end
+
+rule "6: Redact if must redact entry is found"
+	when
+		eval(section.contains("must_redact")==true);
+	then
+		section.redact("name", 6, "must_redact entry was found.");
+		section.redact("address", 6, "must_redact entry was found.");
+	end
+
+
+rule "7: Redact contact information, if applicant is found"
+	when
+		eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant"));
+	then
+		section.redactLineAfter("Name:", "address", 7, "Applicant information was found");
+		section.redactBetween("Address:", "Contact", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Contact point:",  "address", 7, "Applicant information was found");
+		section.redactLineAfter("Phone:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Fax:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Tel.:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Tel:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("E-mail:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Email:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Contact:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Telephone number:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Fax number:", "address", 7, "Applicant information was found");
+		section.redactLineAfter("Telephone:", "address", 7, "Applicant information was found");
+		section.redactBetween("No:", "Fax", "address", 7, "Applicant information was found");
+		section.redactBetween("Contact:", "Tel.:", "address", 7, "Applicant information was found");
+	end
+
+rule "8: Redact contact information, if Producer is found"
+	when
+		eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance"));
+	then
+		section.redactLineAfter("Name:", "address", 8, "Producer was found");
+		section.redactBetween("Address:", "Contact", "address", 8, "Producer was found");
+		section.redactBetween("Contact:", "Phone", "address", 8, "Producer was found");
+		section.redactBetween("Contact:", "Telephone number:", "address", 8, "Producer was found");
+		section.redactBetween("Address:", "Manufacturing", "address", 8, "Producer was found");
+		section.redactLineAfter("Telephone:", "address", 8, "Producer was found");
+		section.redactLineAfter("Phone:", "address", 8, "Producer was found");
+		section.redactLineAfter("Fax:", "address", 8, "Producer was found");
+		section.redactLineAfter("E-mail:", "address", 8, "Producer was found");
+		section.redactLineAfter("Contact:", "address", 8, "Producer was found");
+		section.redactLineAfter("Fax number:", "address", 8, "Producer was found");
+		section.redactLineAfter("Telephone number:", "address", 8, "Producer was found");
+		section.redactLineAfter("Tel:", "address", 8, "Producer was found");
+		section.redactBetween("No:", "Fax", "address", 8, "Producer was found");
+	end
+