Use default color from configuration-service on unknown type

Merge in RED/redaction-service from DEV5 to master

* commit '88e1c5c58ea44dfab15f086ef58f23df897777d6':
  Let Tables know its headlines

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -39,7 +39,7 @@
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.6</version>
+            <version>1.0.12</version>
         </dependency>
         <dependency>
             <groupId>org.drools</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/SearchableText.java

@@ -26,10 +26,10 @@ public class SearchableText {
     }
 
 
-    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInSensitive) {
+    public List<EntityPositionSequence> getSequences(String searchString, boolean caseInsensitive) {
 
         String normalizedSearchString;
-        if (caseInSensitive) {
+        if (caseInsensitive) {
             normalizedSearchString = searchString.toLowerCase();
         } else {
             normalizedSearchString = searchString;
@@ -44,34 +44,34 @@ public class SearchableText {
             TextPositionSequence partMatch = new TextPositionSequence(sequences.get(i).getPage());
             for (int j = 0; j < sequences.get(i).length(); j++) {
 
-                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInSensitive) == ' ' && sequences.get(i - 1)
-                        .charAt(sequences.get(i - 1).length() - 1, caseInSensitive) == ' ' || j > 0 && sequences.get(i)
-                        .charAt(j, caseInSensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInSensitive) == ' ') {
+                if (i > 0 && j == 0 && sequences.get(i).charAt(0, caseInsensitive) == ' ' && sequences.get(i - 1)
+                        .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) == ' ' || j > 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == ' ' && sequences.get(i).charAt(j - 1, caseInsensitive) == ' ') {
                     if (j == sequences.get(i).length() - 1 && counter != 0 && !partMatch.getTextPositions().isEmpty()) {
                         crossSequenceParts.add(partMatch);
                     }
                     continue;
                 }
 
-                if (j == 0 && sequences.get(i).charAt(j, caseInSensitive) != ' ' && i != 0 && sequences.get(i - 1)
+                if (j == 0 && sequences.get(i).charAt(j, caseInsensitive) != ' ' && i != 0 && sequences.get(i - 1)
                         .charAt(sequences.get(i - 1)
-                                .length() - 1, caseInSensitive) != ' ' && searchChars[counter] == ' ') {
+                                .length() - 1, caseInsensitive) != ' ' && searchChars[counter] == ' ') {
                     counter++;
                 }
 
                 if (sequences.get(i)
-                        .charAt(j, caseInSensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
-                        .charAt(j, caseInSensitive) == '-') {
+                        .charAt(j, caseInsensitive) == searchChars[counter] || counter != 0 && sequences.get(i)
+                        .charAt(j, caseInsensitive) == '-') {
 
                     if (counter != 0 || i == 0 && j == 0 || j != 0 && isSeparator(sequences.get(i)
-                            .charAt(j - 1, caseInSensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
+                            .charAt(j - 1, caseInsensitive)) || j == 0 && i != 0 && isSeparator(sequences.get(i - 1)
                             .charAt(sequences.get(i - 1)
-                                    .length() - 1, caseInSensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
-                            .charAt(sequences.get(i - 1).length() - 1, caseInSensitive) != ' ' && sequences.get(i)
-                            .charAt(j, caseInSensitive) != ' ') {
+                                    .length() - 1, caseInsensitive)) || j == 0 && i != 0 && sequences.get(i - 1)
+                            .charAt(sequences.get(i - 1).length() - 1, caseInsensitive) != ' ' && sequences.get(i)
+                            .charAt(j, caseInsensitive) != ' ') {
                         partMatch.add(sequences.get(i).textPositionAt(j));
                         if (!(j == sequences.get(i).length() - 1 && sequences.get(i)
-                                .charAt(j, caseInSensitive) == '-' && searchChars[counter] != '-')) {
+                                .charAt(j, caseInsensitive) == '-' && searchChars[counter] != '-')) {
                             counter++;
                         }
                     }
@@ -81,11 +81,11 @@ public class SearchableText {
 
                         if (i == sequences.size() - 1 && j == sequences.get(i).length() - 1 || j != sequences.get(i)
                                 .length() - 1 && isSeparator(sequences.get(i)
-                                .charAt(j + 1, caseInSensitive)) || j == sequences.get(i)
+                                .charAt(j + 1, caseInsensitive)) || j == sequences.get(i)
                                 .length() - 1 && isSeparator(sequences.get(i + 1)
-                                .charAt(0, caseInSensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
-                                .charAt(j, caseInSensitive) != ' ' && sequences.get(i + 1)
-                                .charAt(0, caseInSensitive) != ' ') {
+                                .charAt(0, caseInsensitive)) || j == sequences.get(i).length() - 1 && sequences.get(i)
+                                .charAt(j, caseInsensitive) != ' ' && sequences.get(i + 1)
+                                .charAt(0, caseInsensitive) != ' ') {
                             finalMatches.addAll(buildEntityPositionSequence(crossSequenceParts));
                         }
 

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -41,6 +41,9 @@ public class DictionaryService {
     @Getter
     private List<String> caseInsensitiveTypes = new ArrayList<>();
 
+    @Getter
+    private float[] defaultColor;
+
 
     public void updateDictionary() {
 
@@ -71,6 +74,7 @@ public class DictionaryService {
                         .map(TypeResult::getType)
                         .collect(Collectors.toList());
                 dictionary = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> convertEntries(s)));
+                defaultColor = dictionaryClient.getDefaultColor().getColor();
             }
         } catch (FeignException e) {
             log.warn("Got some unknown feignException", e);

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -27,6 +27,7 @@ public class EntityRedactionService {
     private final DictionaryService dictionaryService;
     private final DroolsExecutionService droolsExecutionService;
 
+
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
@@ -39,7 +40,6 @@ public class EntityRedactionService {
 
             List<Table> tables = paragraph.getTables();
 
-            List<SearchableText> searchableRows = new ArrayList<>();
             for (Table table : tables) {
                 for (List<Cell> row : table.getRows()) {
                     SearchableText searchableRow = new SearchableText();
@@ -51,13 +51,28 @@ public class EntityRedactionService {
                             searchableRow.addAll(textBlock.getSequences());
                         }
                     }
-                    searchableRows.add(searchableRow);
+                    Set<Entity> rowEntities = findEntities(searchableRow, table.getHeadline());
+
+                    Section analysedRowSection = droolsExecutionService.executeRules(Section.builder()
+                            .entities(rowEntities)
+                            .text(searchableRow.getAsStringWithLinebreaks())
+                            .searchText(searchableRow.toString())
+                            .headline(table.getHeadline())
+                            .build());
+
+                    for (Entity entity : analysedRowSection.getEntities()) {
+                        if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                            entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), true));
+                        } else {
+                            entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), false));
+                        }
+                    }
+                    documentEntities.addAll(analysedRowSection.getEntities());
                 }
             }
 
             Set<Entity> entities = findEntities(searchableText, paragraph.getHeadline());
-            Section analysedSection = droolsExecutionService.executeRules(Section
-                    .builder()
+            Section analysedSection = droolsExecutionService.executeRules(Section.builder()
                     .entities(entities)
                     .text(searchableText.getAsStringWithLinebreaks())
                     .searchText(searchableText.toString())
@@ -65,46 +80,27 @@ public class EntityRedactionService {
                     .build());
 
             for (Entity entity : analysedSection.getEntities()) {
-                if(dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
+                if (dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
                     entity.setPositionSequences(searchableText.getSequences(entity.getWord(), true));
-                } else{
+                } else {
                     entity.setPositionSequences(searchableText.getSequences(entity.getWord(), false));
                 }
             }
 
             documentEntities.addAll(analysedSection.getEntities());
-
-            for (SearchableText searchableRow : searchableRows) {
-                Set<Entity> rowEntities = findEntities(searchableRow, "//TODO TableHeader");
-
-                Section analysedRowSection = droolsExecutionService.executeRules(Section
-                        .builder()
-                        .entities(rowEntities)
-                        .text(searchableRow.getAsStringWithLinebreaks())
-                        .searchText(searchableRow.toString())
-                        .headline("//TODO TableHeader")
-                        .build());
-
-                for (Entity entity : analysedRowSection.getEntities()) {
-                    if(dictionaryService.getCaseInsensitiveTypes().contains(entity.getType())) {
-                        entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), true));
-                    } else{
-                        entity.setPositionSequences(searchableRow.getSequences(entity.getWord(), false));
-                    }
-                }
-                documentEntities.addAll(analysedRowSection.getEntities());
-            }
         }
 
         documentEntities.forEach(entity -> {
             entity.getPositionSequences().forEach(sequence -> {
-                classifiedDoc.getEntities().computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>()).add(
-                        new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List.of(sequence), entity.getHeadline(), entity.getMatchedRule())
-                );
+                classifiedDoc.getEntities()
+                        .computeIfAbsent(sequence.getPageNumber(), (x) -> new HashSet<>())
+                        .add(new Entity(entity.getWord(), entity.getType(), entity.isRedaction(), entity.getRedactionReason(), List
+                                .of(sequence), entity.getHeadline(), entity.getMatchedRule()));
             });
         });
     }
 
+
     private Set<Entity> findEntities(SearchableText searchableText, String headline) {
 
         String inputString = searchableText.toString();
@@ -113,7 +109,7 @@ public class EntityRedactionService {
         Set<Entity> found = new HashSet<>();
         for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
 
-            if(dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())){
+            if (dictionaryService.getCaseInsensitiveTypes().contains(entry.getKey())) {
                 found.addAll(find(lowercaseInputString, entry.getValue(), entry.getKey(), headline));
             } else {
                 found.addAll(find(inputString, entry.getValue(), entry.getKey(), headline));
@@ -126,7 +122,8 @@ public class EntityRedactionService {
     }
 
 
-    private Set<Entity> find(String inputString, Set<String> values, String type, String headline){
+    private Set<Entity> find(String inputString, Set<String> values, String type, String headline) {
+
         Set<Entity> found = new HashSet<>();
         for (String value : values) {
             int startIndex;
@@ -135,9 +132,8 @@ public class EntityRedactionService {
                 startIndex = inputString.indexOf(value, stopIndex);
                 stopIndex = startIndex + value.length();
 
-                if (startIndex > -1 &&
-                        (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString.charAt(startIndex - 1))) &&
-                        (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
+                if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(inputString.charAt(startIndex - 1)) || isSeparator(inputString
+                        .charAt(startIndex - 1))) && (stopIndex == inputString.length() || isSeparator(inputString.charAt(stopIndex)))) {
                     found.add(new Entity(inputString.substring(startIndex, stopIndex), type, startIndex, stopIndex, headline));
                 }
             } while (startIndex > -1);
@@ -146,20 +142,24 @@ public class EntityRedactionService {
     }
 
 
-
     private boolean isSeparator(char c) {
+
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
+
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
             for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                     wordsToRemove.add(inner);
                 }
             }
         }
         entities.removeAll(wordsToRemove);
     }
+
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/segmentation/SectionsBuilderService.java

@@ -17,7 +17,6 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 @SuppressWarnings("all")
 public class SectionsBuilderService {
 
-
     public void buildSections(Document document) {
 
         List<AbstractTextContainer> chunkWords = new ArrayList<>();
@@ -29,7 +28,8 @@ public class SectionsBuilderService {
         for (Page page : document.getPages()) {
             for (AbstractTextContainer current : page.getTextBlocks()) {
 
-                if (current.getClassification() == null || current.getClassification().equals("Header") || current.getClassification().equals("Footer")) {
+                if (current.getClassification() == null || current.getClassification()
+                        .equals("Header") || current.getClassification().equals("Footer")) {
                     continue;
                 }
 
@@ -37,7 +37,7 @@ public class SectionsBuilderService {
 
                 if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
 
-                    Paragraph chunkBlock = buildTextBlock(chunkWords);
+                    Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
                     chunkBlock.setHeadline(lastHeadline);
                     lastHeadline = current.getText();
                     chunkBlockList.add(chunkBlock);
@@ -51,7 +51,7 @@ public class SectionsBuilderService {
             }
         }
 
-        Paragraph chunkBlock = buildTextBlock(chunkWords);
+        Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
         if (chunkBlock != null) {
             chunkBlockList.add(chunkBlock);
             chunkBlock.setHeadline(lastHeadline);
@@ -61,7 +61,7 @@ public class SectionsBuilderService {
     }
 
 
-    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList) {
+    private Paragraph buildTextBlock(List<AbstractTextContainer> wordBlockList, String lastHeadline) {
 
         Paragraph paragraph = new Paragraph();
         TextBlock textBlock = null;
@@ -70,17 +70,23 @@ public class SectionsBuilderService {
         boolean splitByTable = false;
 
         Iterator<AbstractTextContainer> itty = wordBlockList.iterator();
-        boolean alreadyAdded= false;
+        boolean alreadyAdded = false;
+        AbstractTextContainer previous = null;
         while (itty.hasNext()) {
             AbstractTextContainer container = itty.next();
 
             if (container instanceof Table) {
                 splitByTable = true;
 
+                if (previous != null && previous instanceof TextBlock && previous.getText().startsWith("Table ")) {
+                    ((Table) container).setHeadline(previous.getText());
+                } else {
+                    ((Table) container).setHeadline("Table in: " + lastHeadline);
+                }
 
                 if (textBlock != null && !alreadyAdded) {
                     paragraph.getPageBlocks().add(textBlock);
-                    alreadyAdded =true;
+                    alreadyAdded = true;
                 }
                 paragraph.getPageBlocks().add(container);
                 continue;
@@ -89,24 +95,28 @@ public class SectionsBuilderService {
             TextBlock wordBlock = (TextBlock) container;
 
             if (textBlock == null) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else if (splitByTable) {
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
                 alreadyAdded = false;
             } else if (pageBefore != -1 && wordBlock.getPage() != pageBefore) {
                 textBlock.setPage(pageBefore);
                 paragraph.getPageBlocks().add(textBlock);
-                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock.getSequences(), wordBlock.getRotation());
+                textBlock = new TextBlock(wordBlock.getMinX(), wordBlock.getMaxX(), wordBlock.getMinY(), wordBlock.getMaxY(), wordBlock
+                        .getSequences(), wordBlock.getRotation());
                 textBlock.setPage(wordBlock.getPage());
             } else {
                 TextBlock spatialEntity = textBlock.union(wordBlock);
-                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(),
-                        spatialEntity.getWidth(), spatialEntity.getHeight());
+                textBlock.resize(spatialEntity.getMinX(), spatialEntity.getMinY(), spatialEntity.getWidth(), spatialEntity
+                        .getHeight());
             }
             pageBefore = wordBlock.getPage();
             splitByTable = false;
+            previous = container;
         }
 
         if (textBlock != null && !alreadyAdded) {
@@ -115,5 +125,4 @@ public class SectionsBuilderService {
         return paragraph;
     }
 
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/tableextraction/model/Table.java

@@ -13,6 +13,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.tableextraction.utils.Utils;
 
 import lombok.Getter;
+import lombok.Setter;
 
 @SuppressWarnings("all")
 public class Table extends AbstractTextContainer {
@@ -21,6 +22,10 @@ public class Table extends AbstractTextContainer {
 
     private RectangleSpatialIndex<Cell> si = new RectangleSpatialIndex<>();
 
+    @Getter
+    @Setter
+    private String headline;
+
     @Getter
     private int rowCount = 0;
     @Getter

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -203,7 +203,7 @@ public class AnnotationHighlightService {
         }
 
         if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
-            return dictionaryService.getEntryColors().get("default");
+            return dictionaryService.getDefaultColor();
         }
 
         return dictionaryService.getEntryColors().get(entity.getType());

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -35,6 +35,7 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.iqser.red.service.configuration.v1.api.model.DefaultColor;
 import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
@@ -57,7 +58,6 @@ public class RedactionIntegrationTest {
     private static final String ADDRESS_CODE = "address";
     private static final String NAME_CODE = "name";
     private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
-    private static final String DEFAULT = "default";
 
     @Autowired
     private RedactionController redactionController;
@@ -109,7 +109,7 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
         when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
         when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
-        when(dictionaryClient.getDictionaryForType(DEFAULT)).thenReturn(getDictionaryResponse(DEFAULT));
+        when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
     }
 
 
@@ -135,7 +135,6 @@ public class RedactionIntegrationTest {
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
-        dictionary.put(DEFAULT, new ArrayList<>());
     }
 
 
@@ -151,19 +150,16 @@ public class RedactionIntegrationTest {
         typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
         typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
         typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
-        typeColorMap.put(DEFAULT, new float[]{1, 0.502f, 0});
 
         hintTypeMap.put(VERTEBRATES_CODE, true);
         hintTypeMap.put(ADDRESS_CODE, false);
         hintTypeMap.put(NAME_CODE, false);
         hintTypeMap.put(NO_REDACTION_INDICATOR, true);
-        hintTypeMap.put(DEFAULT, true);
 
         caseInSensitiveMap.put(VERTEBRATES_CODE, true);
         caseInSensitiveMap.put(ADDRESS_CODE, false);
         caseInSensitiveMap.put(NAME_CODE, false);
         caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
-        caseInSensitiveMap.put(DEFAULT, true);
     }