Use default color from configuration-service on unknown type

Pull request #15 : Let Tables know its headlines
Merge in RED/redaction-service from DEV5 to master * commit '88e1c5c58ea44dfab15f086ef58f23df897777d6': Let Tables know its headlines
2020-07-28 12:38:59 +02:00 · 2020-07-27 15:45:43 +02:00 · 2020-07-27 15:33:25 +02:00 · 2020-07-27 13:52:40 +02:00 · 2020-07-27 13:39:31 +02:00 · 2020-07-27 13:38:13 +02:00
17 changed files with 427 additions and 214 deletions
@@ -13,6 +13,7 @@ public class RedactionLogEntry {
    private String value;
    private String reason;
    private boolean redacted;
+    private boolean isHint;
    private String section;
    private float[] color;
    private List<Rectangle> positions = new ArrayList<>();
@@ -39,7 +39,7 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.1</version>
+            <version>1.0.12</version>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;

+import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -22,6 +23,7 @@ import org.springframework.context.annotation.Import;
 import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

@Import({DefaultWebMvcConfiguration.class})
@@ -33,24 +35,35 @@ public class Application {
    @Autowired
    private RulesClient rulesClient;

+
    public static void main(String[] args) {
+
        SpringApplication.run(Application.class, args);
    }

+
    @Bean
    public KieContainer kieContainer() {

-        KieServices kieServices = KieServices.Factory.get();
+        try {
+            KieServices kieServices = KieServices.Factory.get();

-        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-        RulesResponse rules = rulesClient.getRules();
-        InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
-        kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
-        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-        kieBuilder.buildAll();
-        KieModule kieModule = kieBuilder.getKieModule();
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            RulesResponse rules = rulesClient.getRules();
+            if (StringUtils.isEmpty(rules.getRules())) {
+                throw new RuntimeException("Rules cannot be empty.");
+            }
+            InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();

-        return kieServices.newKieContainer(kieModule.getReleaseId());
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
+        }

    }

@@ -5,6 +5,6 @@ import org.springframework.cloud.openfeign.FeignClient;
 import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
 import com.iqser.red.service.configuration.v1.api.resource.RulesResource;

-@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
 public interface DictionaryClient extends DictionaryResource {
 }
@@ -39,6 +39,12 @@ public class TextPositionSequence implements CharSequence {
        return text.charAt(0);
    }

+    public char charAt(int index, boolean caseInSensitive) {
+        TextPosition textPosition = textPositionAt(index);
+        String text = textPosition.getUnicode();
+        return caseInSensitive ? text.toLowerCase().charAt(0) : text.charAt(0);
+    }
+
    @Override
    public TextPositionSequence subSequence(int start, int end) {
        return new TextPositionSequence(textPositions.subList(start, end), page);
@@ -13,45 +13,65 @@ public class SearchableText {

    private List<TextPositionSequence> sequences = new ArrayList<>();

+
    public void add(TextPositionSequence textPositionSequence) {
+
        sequences.add(textPositionSequence);
    }

+
    public void addAll(List<TextPositionSequence> textPositionSequences) {
+
        sequences.addAll(textPositionSequences);
    }


-    public List<EntityPositionSequence> getSequences(String searchString) {
+    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {

-        char[] searchChars = searchString.replaceAll("\\n", " ").toCharArray();
+        String normalizedSearchString;
+        if (caseInsensitive) {
+            normalizedSearchString = searchString.toLowerCase();
+        } else {
+            normalizedSearchString = searchString;
+        }
+
+        char[] searchChars = normalizedSearchString.replaceAll("\\n", " ").toCharArray();
        int counter = 0;

-
        List<TextPositionSequence> crossSequenceParts = new ArrayList<>();
        List<EntityPositionSequence> finalMatches = new ArrayList<>();
        for (int i = 0; i < sequences.size(); i++) {
            TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
            for (int j = 0; j < sequences.get(i).length(); j++) {

-                if(i > 0 && j == 0 && sequences.get(i).charAt(0) == ' ' && sequences.get(i - 1).charAt(sequences.get(i - 1).length() - 1) == ' '
-                || j > 0 && sequences.get(i).charAt(j) == ' ' && sequences.get(i).charAt(j - 1) == ' '){
-                    if(j == sequences.get(i).length() -1  && counter != 0 && !partMatch.getTextPositions().isEmpty()){
+                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInsensitive) == ' ' && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInsensitive) == ' ') {
+                    if (j == sequences.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
                        crossSequenceParts.add(partMatch);
                    }
                    continue;
                }

-                if(j == 0 && sequences.get(i).charAt(j) != ' ' && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && searchChars[counter] == ' '){
+                if (j == 0 && sequences.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1)
+                                .length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
                    counter++;
                }

-                if (sequences.get(i).charAt(j) == searchChars[counter] || counter != 0 && sequences.get(i).charAt(j) == '-') {
+                if (sequences.get(i)
+                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == '-') {

-                    if(counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i).charAt(j - 1)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1))
-                            || j == 0 && i != 0 && sequences.get(i - 1).charAt(sequences.get(i - 1).length() -1) != ' ' && sequences.get(i).charAt(j) != ' ') {
+                    if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1)
+                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) != ' ' && sequences.get(i)
+                            .charAt(j, caseInsensitive) != ' ') {
                        partMatch.add(sequences.get(i).textPositionAt(j));
-                        if (!(j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) == '-' && searchChars[counter] != '-')) {
+                        if (!(j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                            counter++;
                        }
                    }
@@ -59,10 +79,13 @@ public class SearchableText {
                    if (counter == searchString.length()) {
                        crossSequenceParts.add(partMatch);

-                        if(i == sequences.size() - 1 && j == sequences.get(i).length() -1
-                                || j != sequences.get(i).length() -1 && isSeparator(sequences.get(i).charAt(j +1))
-                                || j == sequences.get(i).length() -1 && isSeparator(sequences.get(i + 1).charAt(0))
-                                || j == sequences.get(i).length() -1 && sequences.get(i).charAt(j) != ' ' && sequences.get(i + 1).charAt(0) != ' ') {
+                        if (i == sequences.size() - 1 && j == sequences.get(i).length() - 1 || j != sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i)
+                                .charAt(j + 1, caseInsensitive)) || j == sequences.get(i)
+                                .length() - 1 && isSeparator(sequences.get(i + 1)
+                                .charAt(0, caseInsensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && sequences.get(i + 1)
+                                .charAt(0, caseInsensitive) != ' ') {
                            finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
                        }

@@ -72,14 +95,14 @@ public class SearchableText {
                    }
                } else {
                    counter = 0;
-                    if(!crossSequenceParts.isEmpty()){
+                    if (!crossSequenceParts.isEmpty()) {
                        j--;
                    }
                    crossSequenceParts = new ArrayList<>();
                    partMatch = new TextPositionSequence(sequences.get(i).getPage());
                }

-                if(j == sequences.get(i).length() -1  && counter != 0){
+                if (j == sequences.get(i).length() - 1 && counter != 0) {
                    crossSequenceParts.add(partMatch);
                }
            }
@@ -89,18 +112,18 @@ public class SearchableText {
    }


-    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts){
+    private List<EntityPositionSequence> buildEntityPositionSequence(List<TextPositionSequence> crossSequenceParts) {

        UUID id = UUID.randomUUID();
        List<EntityPositionSequence> result = new ArrayList<>();
        int currentPage = -1;
        EntityPositionSequence entityPositionSequence = new EntityPositionSequence(id);
-        for (TextPositionSequence textPositionSequence :crossSequenceParts){
-            if(currentPage == -1){
+        for (TextPositionSequence textPositionSequence : crossSequenceParts) {
+            if (currentPage == -1) {
                currentPage = textPositionSequence.getPage();
                entityPositionSequence.setPageNumber(currentPage);
                entityPositionSequence.getSequences().add(textPositionSequence);
-            } else if(currentPage == textPositionSequence.getPage()){
+            } else if (currentPage == textPositionSequence.getPage()) {
                entityPositionSequence.getSequences().add(textPositionSequence);
            } else {
                result.add(entityPositionSequence);
@@ -114,13 +137,14 @@ public class SearchableText {


    private boolean isSeparator(char c) {
+
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }


-
    @Override
    public String toString() {
+
        StringBuilder sb = new StringBuilder();

        TextPositionSequence previous = null;
@@ -137,10 +161,14 @@ public class SearchableText {
            previous = word;
        }

-        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString()).replaceAll("\n", " ").replaceAll("  ", " ");
+        return TextNormalizationUtilities.removeHyphenLineBreaks(sb.toString())
+                .replaceAll("\n", " ")
+                .replaceAll("  ", " ");
    }

-    public String getAsStringWithLinebreaks(){
+
+    public String getAsStringWithLinebreaks() {
+
        StringBuilder sb = new StringBuilder();

        TextPositionSequence previous = null;
@@ -25,13 +25,17 @@ public class Section {

    private String headline;

+
    public boolean contains(String type) {
+
        return entities.stream().anyMatch(entity -> entity.getType().equals(type));
    }

-    public void redact(String type, int ruleNumber, String reason){
+
+    public void redact(String type, int ruleNumber, String reason) {
+
        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -39,9 +43,11 @@ public class Section {
        });
    }

-    public void redactNot(String type, int ruleNumber, String reason){
+
+    public void redactNot(String type, int ruleNumber, String reason) {
+
        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                entity.setRedaction(false);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -49,27 +55,19 @@ public class Section {
        });
    }

-    public void highlightAll(String type){
-        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
-                entity.setRedaction(true);
-            }
-        });
-    }

-
-    public void redactLineAfter(String start, String asType, int ruleNumber, String reason){
+    public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {

        String value = StringUtils.substringBetween(text, start, "\n");

-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
            entities.addAll(found);
        }

        // TODO No need to iterate
        entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -79,19 +77,18 @@ public class Section {
    }


-
-    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason){
+    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason) {

        String value = StringUtils.substringBetween(searchText, start, stop);

-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
            entities.addAll(found);
        }

        // TODO No need to iterate
        entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -100,25 +97,21 @@ public class Section {
    }


-
-
    private Set<Entity> findEntity(String value, String asType) {

        Set<Entity> found = new HashSet<>();

-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = searchText.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();
-
-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText.charAt(startIndex - 1))) &&
-                            (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
-                        found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
-                    }
-                } while (startIndex > -1);
+        int startIndex;
+        int stopIndex = 0;
+        do {
+            startIndex = searchText.indexOf(value, stopIndex);
+            stopIndex = startIndex + value.length();

+            if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText
+                    .charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
+            }
+        } while (startIndex > -1);

        removeEntitiesContainedInLarger(found);

@@ -127,14 +120,18 @@ public class Section {


    private boolean isSeparator(char c) {
+
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

+
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
            for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                    wordsToRemove.add(inner);
                }
            }
@@ -1,6 +1,9 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -32,6 +35,16 @@ public class DictionaryService {
    @Getter
    private Map<String, float[]> entryColors = new HashMap<>();

+    @Getter
+    private List<String> hintTypes = new ArrayList<>();
+
+    @Getter
+    private List<String> caseInsensitiveTypes = new ArrayList<>();
+
+    @Getter
+    private float[] defaultColor;
+
+
    public void updateDictionary() {

        long version = dictionaryClient.getVersion();
@@ -41,16 +54,44 @@ public class DictionaryService {
        }
    }

+
    private void updateDictionaryEntry() {
+
        try {
            TypeResponse typeResponse = dictionaryClient.getAllTypes();
-            if (typeResponse != null && !CollectionUtils.isEmpty(typeResponse.getTypes())) {
-                entryColors = typeResponse.getTypes().stream().collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
-                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> dictionaryClient.getDictionaryForType(s).getEntries().stream().collect(Collectors.toSet())));
+            if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
+                entryColors = typeResponse.getTypes()
+                        .stream()
+                        .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
+                hintTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isHint)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                caseInsensitiveTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isCaseInsensitive)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> convertEntries(s)));
+                defaultColor = dictionaryClient.getDefaultColor().getColor();
            }
        } catch (FeignException e) {
            log.warn("Got some unknown feignException", e);
            throw e;
        }
    }
+
+
+    private Set<String> convertEntries(String s) {
+        if (caseInsensitiveTypes.contains(s)) {
+            return dictionaryClient.getDictionaryForType(s)
+                    .getEntries()
+                    .stream()
+                    .map(String::toLowerCase)
+                    .collect(Collectors.toSet());
+        }
+        return new HashSet<>(dictionaryClient.getDictionaryForType(s).getEntries());
+    }
+
 }
@@ -68,7 +68,7 @@ public class DroolsExecutionService {
            KieModule kieModule = kieBuilder.getKieModule();
            kieContainer.updateToVersion(kieModule.getReleaseId());
        } catch (Exception e) {
-            throw new RulesValidationException("Could not update rules", e);
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
        }

    }
@@ -27,6 +27,7 @@ public class EntityRedactionService {
    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;

+
    public void processDocument(Document classifiedDoc) {

        dictionaryService.updateDictionary();
@@ -39,7 +40,6 @@ public class EntityRedactionService {

            List<Table> tables = paragraph.getTables();

-            List<SearchableText> searchableRows = new ArrayList<>();
            for (Table table : tables) {
                for (List<Cell> row : table.getRows()) {
                    SearchableText searchableRow = new SearchableText();
@@ -51,13 +51,28 @@ public class EntityRedactionService {
                            searchableRow.addAll(textBlock.getSequences());
                        }
                    }
-                    searchableRows.add(searchableRow);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline());
+
+                    Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
+                            .entities(rowEntities)
+                            .text(searchableRow.getAsStringWithLinebreaks())
+                            .searchText(searchableRow.toString())
+                            .headline(table.getHeadline())
+                            .build());
+
+                    for (Entity entity : analysedRowSection.getEntities()) {
+                        if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                            entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), true));
+                        } else {
+                            entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), false));
+                        }
+                    }
+                    documentEntities.addAll(analysedRowSection.getEntities());
                }
            }

            Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline());
-            Section analysedSection = droolsExecutionService.executeRules(Section
-                    .builder()
+            Section analysedSection = droolsExecutionService.executeRules(Section.builder()
                    .entities(entities)
                    .text(searchableText.getAsStringWithLinebreaks())
                    .searchText(searchableText.toString())
@@ -65,57 +80,39 @@ public class EntityRedactionService {
                    .build());

            for (Entity entity : analysedSection.getEntities()) {
-                entity.setPositionSequences(searchableText.getSequences(entity.getWord()));
+                if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                    entity.setPositionSequences(searchableText.getSequences(entity.getWord(), true));
+                } else {
+                    entity.setPositionSequences(searchableText.getSequences(entity.getWord(), false));
+                }
            }

            documentEntities.addAll(analysedSection.getEntities());
-
-            for (SearchableText searchableRow : searchableRows) {
-                Set<Entity> rowEntities = findEntities(searchableRow, "//TODO TableHeader");
-
-                Section analysedRowSection = droolsExecutionService.executeRules(Section
-                        .builder()
-                        .entities(rowEntities)
-                        .text(searchableRow.getAsStringWithLinebreaks())
-                        .searchText(searchableRow.toString())
-                        .headline("//TODO TableHeader")
-                        .build());
-
-                for (Entity entity : analysedRowSection.getEntities()) {
-                    entity.setPositionSequences(searchableRow.getSequences(entity.getWord()));
-                }
-                documentEntities.addAll(analysedRowSection.getEntities());
-            }
        }

        documentEntities.forEach(entity -> {
            entity.getPositionSequences().forEach(sequence -> {
-                classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
-                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence), entity.getHeadline(), entity.getMatchedRule())
-                );
+                classifiedDoc.getEntities()
+                        .computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>())
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List
+                                .of(sequence), entity.getHeadline(), entity.getMatchedRule()));
            });
        });
    }

+
    private Set<Entity> findEntities(SearchableText searchableText, String headline) {

-        String normalizedInputString = searchableText.toString();
+        String inputString = searchableText.toString();
+        String lowercaseInputString = inputString.toLowerCase();

        Set<Entity> found = new HashSet<>();
        for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
-            for (String value : entry.getValue()) {
-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = normalizedInputString.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();

-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(normalizedInputString.charAt(startIndex - 1)) || isSeparator(normalizedInputString.charAt(startIndex - 1))) &&
-                            (stopIndex == normalizedInputString.length() || isSeparator(normalizedInputString.charAt(stopIndex)))) {
-                        found.add(new Entity(normalizedInputString.substring(startIndex, stopIndex), entry.getKey(), startIndex, stopIndex, headline));
-                    }
-                } while (startIndex > -1);
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())) {
+                found.addAll(find(lowercaseInputString, entry.getValue(), entry.getKey(), headline));
+            } else {
+                found.addAll(find(inputString, entry.getValue(), entry.getKey(), headline));
            }
        }

@@ -124,19 +121,45 @@ public class EntityRedactionService {
        return found;
    }

+
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline) {
+
+        Set<Entity> found = new HashSet<>();
+        for (String value : values) {
+            int startIndex;
+            int stopIndex = 0;
+            do {
+                startIndex = inputString.indexOf(value, stopIndex);
+                stopIndex = startIndex + value.length();
+
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                    found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline));
+                }
+            } while (startIndex > -1);
+        }
+        return found;
+    }
+
+
    private boolean isSeparator(char c) {
+
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

+
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
            for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                    wordsToRemove.add(inner);
                }
            }
        }
        entities.removeAll(wordsToRemove);
    }
+
 }
@@ -17,7 +17,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@SuppressWarnings("all")
 public class SectionsBuilderService {

-
    public void buildSections(Document document) {

        List<AbstractTextContainer> chunkWords = new ArrayList<>();
@@ -29,7 +28,8 @@ public class SectionsBuilderService {
        for (Page page : document.getPages()) {
            for (AbstractTextContainer current : page.getTextBlocks()) {

-                if (current.getClassification() == null || current.getClassification().equals("Header") || current.getClassification().equals("Footer")) {
+                if (current.getClassification() == null || current.getClassification()
+                        .equals("Header") || current.getClassification().equals("Footer")) {
                    continue;
                }

@@ -37,7 +37,7 @@ public class SectionsBuilderService {

                if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {

-                    Paragraph chunkBlock = buildTextBlock(chunkWords);
+                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                    chunkBlock.setHeadline(lastHeadline);
                    lastHeadline = current.getText();
                    chunkBlockList.add(chunkBlock);
@@ -51,7 +51,7 @@ public class SectionsBuilderService {
            }
        }

-        Paragraph chunkBlock = buildTextBlock(chunkWords);
+        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
        if (chunkBlock != null) {
            chunkBlockList.add(chunkBlock);
            chunkBlock.setHeadline(lastHeadline);
@@ -61,7 +61,7 @@ public class SectionsBuilderService {
    }


-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList) {
+    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {

        Paragraph paragraph = new Paragraph();
        TextBlock textBlock = null;
@@ -70,17 +70,23 @@ public class SectionsBuilderService {
        boolean splitByTable = false;

        Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
-        boolean alreadyAdded= false;
+        boolean alreadyAdded = false;
+        AbstractTextContainer previous = null;
        while (itty.hasNext()) {
            AbstractTextContainer container = itty.next();

            if (container instanceof Table) {
                splitByTable = true;

+                if (previous != null && previous instanceof TextBlock && previous.getText().startsWith("Table ")) {
+                    ((Table) container).setHeadline(previous.getText());
+                } else {
+                    ((Table) container).setHeadline("Table in: " + lastHeadline);
+                }

                if (textBlock != null && !alreadyAdded) {
                    paragraph.getPageBlocks().add(textBlock);
-                    alreadyAdded =true;
+                    alreadyAdded = true;
                }
                paragraph.getPageBlocks().add(container);
                continue;
@@ -89,24 +95,28 @@ public class SectionsBuilderService {
            TextBlock wordBlock = (TextBlock) container;

            if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
            } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
                alreadyAdded = false;
            } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                textBlock.setPage(pageBefore);
                paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                textBlock.setPage(wordBlock.getPage());
            } else {
                TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
-                        spatialEntity.getWidth(), spatialEntity.getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                        .getHeight());
            }
            pageBefore = wordBlock.getPage();
            splitByTable = false;
+            previous = container;
        }

        if (textBlock != null && !alreadyAdded) {
@@ -115,5 +125,4 @@ public class SectionsBuilderService {
        return paragraph;
    }

-
 }
@@ -13,6 +13,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;

 import lombok.Getter;
+import lombok.Setter;

@SuppressWarnings("all")
 public class Table extends AbstractTextContainer {
@@ -21,6 +22,10 @@ public class Table extends AbstractTextContainer {

    private RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();

+    @Getter
+    @Setter
+    private String headline;
+
    @Getter
    private int rowCount = 0;
    @Getter
@@ -4,6 +4,7 @@ import java.awt.Color;
 import java.io.IOException;
 import java.util.List;

+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -36,8 +37,10 @@ import lombok.extern.slf4j.Slf4j;
@Service
@RequiredArgsConstructor
 public class AnnotationHighlightService {
+
    private final DictionaryService dictionaryService;

+
    public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {

        for (int page = 1; page <= document.getNumberOfPages(); page++) {
@@ -98,18 +101,29 @@ public class AnnotationHighlightService {
                            posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
                            posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
                            posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 4;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() - height + 4;
                        } else {

                            posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
-                            posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
-                            posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj() + 2;
+                            posXEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() + textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getWidth() + 1;
+                            posYInit = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getYDirAdj() + 2;
                        }

                        Rectangle textHighlightRectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);

-
                        List<PDAnnotation> annotations = pdPage.getAnnotations();
                        PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
                        highlight.constructAppearances();
@@ -121,10 +135,11 @@ public class AnnotationHighlightService {
                        annotationPosition.setUpperRightY(posYEnd + height);

                        highlight.setRectangle(annotationPosition);
-                        if (!flatRedaction) {
+                        if (!flatRedaction && !isHint(entity)) {
                            highlight.setAnnotationName(entityPositionSequence.getId().toString());
                            highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity.getHeadline() + "\"");
+                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity
+                                    .getHeadline() + "\"");
                        }

                        highlight.setQuadPoints(toQuadPoints(textHighlightRectangle));
@@ -150,6 +165,7 @@ public class AnnotationHighlightService {
                redactionLogEntry.setType(entity.getType());
                redactionLogEntry.setRedacted(entity.isRedaction());
                redactionLogEntry.setSection(entity.getHeadline());
+                redactionLogEntry.setHint(isHint(entity));
                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
            }

@@ -161,32 +177,46 @@ public class AnnotationHighlightService {

        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
        // of the area to be highlighted
-        return new float[]{
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
+        return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
+                .getY() + rectangle.getHeight(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
    }


    private boolean isRedactionType(Entity entity) {
+
        if (!entity.isRedaction()) {
            return false;
        }
-        if(entity.getType().equalsIgnoreCase("VERTEBRATE") || entity.getType().equalsIgnoreCase("NO_REDACTION_INDICATOR") ){
-            // TODO in RED-161.
+        if (isHint(entity)) {
            return false;
        }
-        return dictionaryService.getDictionary().keySet().contains(entity.getType());
+        return true;
    }

+
    private float[] getColor(Entity entity) {
-        if (!entity.isRedaction()) {
+
+        if (!entity.isRedaction() && !isHint(entity)) {
            return new float[]{0.627f, 0.627f, 0.627f};
        }
+
+        if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
+            return dictionaryService.getDefaultColor();
+        }
+
        return dictionaryService.getEntryColors().get(entity.getType());
    }

+    private boolean isHint(Entity entity) {
+        List<String> hintTypes = dictionaryService.getHintTypes();
+        if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
+            return true;
+        }
+        return false;
+    }
+
    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {

        contentStream.setStrokingColor(Color.LIGHT_GRAY);
@@ -211,13 +241,15 @@ public class AnnotationHighlightService {


    private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+
        for (List<Cell> row : table.getRows()) {
            for (Cell cell : row) {

                if (cell != null) {
                    contentStream.setLineWidth(0.5f);
                    contentStream.setStrokingColor(Color.CYAN);
-                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell.getHeight());
+                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
+                            .getHeight());
                    contentStream.stroke();

 //                    contentStream.setStrokingColor(Color.GREEN);
@@ -242,4 +274,5 @@ public class AnnotationHighlightService {
            contentStream.endText();
        }
    }
+
 }
@@ -12,9 +12,9 @@ import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.HashMap;
 import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
@@ -33,11 +33,9 @@ import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
-
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-
 import org.springframework.test.context.junit4.SpringRunner;

+import com.iqser.red.service.configuration.v1.api.model.DefaultColor;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -47,6 +45,7 @@ import com.iqser.red.service.redaction.v1.model.RedactionResult;
 import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

@Ignore
@@ -54,10 +53,11 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
@SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {

-    public static final String VERTEBRATES_CODE = "VERTEBRATE";
-    public static final String ADDRESS_CODE = "ADDRESS";
-    public static final String NAME_CODE = "NAME";
-    public static final String NO_REDACTION_INDICATOR = "NO_REDACTION_INDICATOR";
+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String VERTEBRATES_CODE = "vertebrate";
+    private static final String ADDRESS_CODE = "address";
+    private static final String NAME_CODE = "name";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";

    @Autowired
    private RedactionController redactionController;
@@ -68,8 +68,10 @@ public class RedactionIntegrationTest {
    @MockBean
    private DictionaryClient dictionaryClient;

-    private Map<String, List<String>> dictionary = new HashMap<>();
-    private Map<String, float[]> typeColorMap = new HashMap<>();
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, float[]> typeColorMap = new HashMap<>();
+    private final Map<String, Boolean> hintTypeMap = new HashMap<>();
+    private final Map<String, Boolean> caseInSensitiveMap = new HashMap<>();

    @TestConfiguration
    public static class RedactionIntegrationTestConfiguration {
@@ -80,21 +82,24 @@ public class RedactionIntegrationTest {
            KieServices kieServices = KieServices.Factory.get();

            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
-            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
            kieBuilder.buildAll();
            KieModule kieModule = kieBuilder.getKieModule();

            return kieServices.newKieContainer(kieModule.getReleaseId());
        }
+
    }

+
    @Before
    public void stubRulesClient() {

        when(rulesClient.getVersion()).thenReturn(0L);
-        when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(RULES));

        loadDictionaryForTest();
        loadTypeForTest();
@@ -104,41 +109,95 @@ public class RedactionIntegrationTest {
        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
+        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
    }

+
    private void loadDictionaryForTest() {
-        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+
+        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/names.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/addresses.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
    }

+
    private String cleanDictionaryEntry(String entry) {
+
        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
    }

+
    private void loadTypeForTest() {
-        typeColorMap.put("VERTEBRATE", new float[]{0, 1, 0});
-        typeColorMap.put("ADDRESS", new float[]{0, 1, 1});
-        typeColorMap.put("NAME", new float[]{1, 1, 0});
-        typeColorMap.put("NO_REDACTION_INDICATOR", new float[]{1, 0.502f, 0});
+
+        typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
+        typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
+        typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
+
+        hintTypeMap.put(VERTEBRATES_CODE, true);
+        hintTypeMap.put(ADDRESS_CODE, false);
+        hintTypeMap.put(NAME_CODE, false);
+        hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+
+        caseInSensitiveMap.put(VERTEBRATES_CODE, true);
+        caseInSensitiveMap.put(ADDRESS_CODE, false);
+        caseInSensitiveMap.put(NAME_CODE, false);
+        caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
    }

+
    private List<TypeResult> getTypeResponse() {
-        return typeColorMap.entrySet().stream().map(typeColor -> TypeResult.builder().type(typeColor.getKey()).color(typeColor.getValue()).build()).collect(Collectors.toList());
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> TypeResult.builder()
+                        .type(typeColor.getKey())
+                        .color(typeColor.getValue())
+                        .isHint(hintTypeMap.get(typeColor.getKey()))
+                        .isCaseInsensitive(caseInSensitiveMap.get(typeColor.getKey()))
+                        .build())
+
+                .collect(Collectors.toList());
    }

+
    private DictionaryResponse getDictionaryResponse(String type) {
-        return DictionaryResponse.builder().color(typeColorMap.get(type)).entries(dictionary.get(type)).build();
+
+        return DictionaryResponse.builder()
+                .color(typeColorMap.get(type))
+                .entries(dictionary.get(type))
+                .isHint(hintTypeMap.get(type))
+                .isCaseInsensitive(caseInSensitiveMap.get(type))
+                .build();
    }

+
    @Test
    public void redactionTest() throws IOException {

        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
        request.setFlatRedaction(false);

        RedactionResult result = redactionController.redact(request);
@@ -152,12 +211,15 @@ public class RedactionIntegrationTest {
        System.out.println("numberOfPages: " + result.getNumberOfPages());
    }

+
    @Test
    public void classificationTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.classify(request);

@@ -166,12 +228,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void sectionsTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.sections(request);

@@ -180,12 +245,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void htmlTablesTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.htmlTables(request);

@@ -194,12 +262,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void htmlTableRotationTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.htmlTables(request);

@@ -208,10 +279,10 @@ public class RedactionIntegrationTest {
        }
    }

-    private String loadFromClassPath(String path) {

-        URL resource = ResourceLoader.class.getClassLoader()
-                .getResource(path);
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
        if (resource == null) {
            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
        }
@@ -219,12 +290,12 @@ public class RedactionIntegrationTest {
            StringBuilder sb = new StringBuilder();
            String str;
            while ((str = br.readLine()) != null) {
-                sb.append(str)
-                        .append("\n");
+                sb.append(str).append("\n");
            }
            return sb.toString();
        } catch (IOException e) {
            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
        }
    }
-}
+
+}
@@ -1,3 +1 @@
-In Vitro
-In vitro
-in vitro
+In Vitro
@@ -100,15 +100,11 @@ Pseudacris triseriata
 poecilia reticulata
 poultry
 quail
-rabbit
-rabbits
 rainbow trout
 Rana limnocharis
 rana
 limnocharis
 rana pipiens
-rat
-rats
 reptile
 reptiles
 ricefish
@@ -4,39 +4,31 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Section

 global Section section

-rule "0: Highlight Indicators"
-	when
-		eval(section.getEntities().isEmpty()==false);
-	then
-		section.highlightAll("VERTEBRATE");
-		section.highlightAll("NO_REDACTION_INDICATOR");
-	end
-

 rule "1: Redacted because Section contains Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==true);
+		eval(section.contains("vertebrate")==true);
 	then
-		section.redact("NAME", 1, "Redacted because Section contains Vertebrate");
-		section.redact("ADDRESS", 1, "Redacted because Section contains Vertebrate");
+		section.redact("name", 1, "Redacted because Section contains Vertebrate");
+		section.redact("address", 1, "Redacted because Section contains Vertebrate");
 	end


 rule "2: Not Redacted because Section contains no Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==false);
+		eval(section.contains("vertebrate")==false);
 	then
-		section.redactNot("NAME", 2, "Not Redacted because Section contains no Vertebrate");
-		section.redactNot("ADDRESS", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate");
 	end


 rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
 	when
-		eval(section.contains("VERTEBRATE")==true && section.contains("NO_REDACTION_INDICATOR")==true);
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true);
 	then
-		section.redactNot("NAME", 3, "Vertebrate was found, but also a no redaction indicator");
-		section.redactNot("ADDRESS", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator");
 	end


@@ -44,12 +36,12 @@ rule "4: Redact contact information, if applicant is found"
 	when
 		eval(section.getText().toLowerCase().contains("applicant"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Contact point:",  "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "ADDRESS", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4");
+		section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Contact point:",  "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4");
 	end


@@ -57,10 +49,10 @@ rule "5: Redact contact information, if 'Producer of the plant protection produc
 	when
 		eval(section.getText().contains("Producer of the plant protection product"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Contact:", "Phone", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Phone:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Fax:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("E-mail:", "ADDRESS", 5, "xxxx");
+		section.redactLineAfter("Name:", "address", 5, "xxxx");
+		section.redactBetween("Address:", "Contact", "address", 5, "xxxx");
+		section.redactBetween("Contact:", "Phone", "address", 5, "xxxx");
+		section.redactLineAfter("Phone:", "address", 5, "xxxx");
+		section.redactLineAfter("Fax:", "address", 5, "xxxx");
+		section.redactLineAfter("E-mail:", "address", 5, "xxxx");
 	end
Author	SHA1	Message	Date
deiflaender	872c384dc6	Use default color from configuration-service on unknown type	2020-07-28 12:38:59 +02:00
Dominique Eiflaender	2aca35e5a0	Pull request #15 : Let Tables know its headlines Merge in RED/redaction-service from DEV5 to master * commit '88e1c5c58ea44dfab15f086ef58f23df897777d6': Let Tables know its headlines	2020-07-27 15:45:43 +02:00
deiflaender	88e1c5c58e	Let Tables know its headlines	2020-07-27 15:33:25 +02:00
Dominique Eiflaender	b7ee62f44d	Pull request #14 : RED-207: Match caseInsensitive dictionaries caseInSensitive Merge in RED/redaction-service from RED-207 to master * commit '135a715e22e6c2536b268db29161552cfd7a6c1c': Fixed style in EnityRedactionService Fixed wrong naming of caseInsensitive RED-207: Match caseInsensitive dictionaries caseInSensitive	2020-07-27 13:52:40 +02:00
deiflaender	135a715e22	Fixed style in EnityRedactionService	2020-07-27 13:39:31 +02:00
deiflaender	c953f161b2	Fixed wrong naming of caseInsensitive	2020-07-27 13:38:13 +02:00
deiflaender	f0e48087ff	RED-207: Match caseInsensitive dictionaries caseInSensitive	2020-07-27 13:20:00 +02:00
Cheng Zhu	d282680cc8	Pull request #13 : Use Hint in IntegrationTest Merge in RED/redaction-service from DEV3 to master * commit 'b57a4a2db3fb8090004dd5f3babd0d7786312730': Use Hint in IntegrationTest	2020-07-24 11:03:58 +02:00
deiflaender	b57a4a2db3	Use Hint in IntegrationTest	2020-07-24 10:45:04 +02:00
Thierry Goeckel	ca439d821d	Pull request #12 : Bugfix/RED-183 Merge in RED/redaction-service from bugfix/RED-183 to master * commit '7dbe03483b7a8a3c391e9d0791717b9fab5fc5db': RED-183: Fix catching validation errors Fix style.	2020-07-23 14:48:11 +02:00
Thierry Göckel	7dbe03483b	RED-183: Fix catching validation errors	2020-07-23 13:36:02 +02:00
Thierry Göckel	33ab09d7fc	Fix style.	2020-07-23 13:35:58 +02:00
Thierry Goeckel	9a425a8594	Pull request #11 : Feature/RED-161: replace hard-coded hint-type with dictionary-service query Merge in RED/redaction-service from feature/RED-161 to master * commit '3eb37fa6db0d9f1a4e6b93b2047b3b96c94cd9ff': RED-161: reverse caseSensitive RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-23 10:59:37 +02:00
cheng	3eb37fa6db	RED-161: reverse caseSensitive	2020-07-23 10:51:30 +02:00
cheng	c8375a1ad9	RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-22 22:54:05 +02:00
Dominique Eiflaender	94827a417f	Pull request #10 : Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false Merge in RED/redaction-service from RedactionLog1 to master * commit 'e7ea12a1b03f7f79e0f4fa12671f91ee326eef73': CodeStyle Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:15:56 +02:00
deiflaender	e7ea12a1b0	CodeStyle	2020-07-22 14:14:10 +02:00
deiflaender	6a9df397eb	Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:06:45 +02:00
Cheng Zhu	41c61dd214	Pull request #9 : DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming. Merge in RED/redaction-service from Fix1 to master * commit '49fe418a60f424e373038b1f7b9d43ff4afde37c': DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:20:22 +02:00
deiflaender	49fe418a60	DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:17:52 +02:00
Dominique Eiflaender	c98f1d5e37	Pull request #8 : RED-169: Fixed startup problem Merge in RED/redaction-service from RED-169 to master * commit 'c0882346a9d1554c51ccc526d1f6d7c277ffbc7c': RED-169: Fixed startup problem	2020-07-21 14:53:55 +02:00
deiflaender	c0882346a9	RED-169: Fixed startup problem	2020-07-21 14:49:00 +02:00