REd-106: enable dictionary version

redaction-service-v1/redaction-service-server-v1/pom.xml

@@ -39,7 +39,7 @@
         <dependency>
             <groupId>com.iqser.red.service</groupId>
             <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.1</version>
         </dependency>
         <dependency>
             <groupId>org.drools</groupId>

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/client/DictionaryClient.java

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = RulesResource.SERVICE_NAME, url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface DictionaryClient extends DictionaryResource {
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/DictionaryService.java

@@ -1,58 +1,58 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;
 
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import javax.annotation.PostConstruct;
-
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;
 
-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 
+import feign.FeignException;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 
-@Slf4j
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class DictionaryService {
 
-    public static final String VERTEBRATES_CODE = "VERTEBRATE";
-    public static final String ADDRESS_CODE = "ADDRESS";
-    public static final String NAME_CODE = "NAME";
-    public static final String NO_REDACTION_INDICATOR = "NO_REDACTION_INDICATOR";
+    private final DictionaryClient dictionaryClient;
+
+    private long dictionaryVersion = -1;
 
     @Getter
-    private Map<String, Set<String>> dictionary = new HashMap<>();
+    private Map<String, Set<String>> dictionaryEntry = new HashMap<>();
 
     @Getter
-    private long generation;
-
-    @PostConstruct
-    public void init() {
-        loadFromResourceFiles();
-    }
-
+    private Map<String, float[]> entryColors = new HashMap<>();
 
     public void updateDictionary() {
-        //TODO
+
+        long version = dictionaryClient.getVersion();
+        if (version > dictionaryVersion) {
+            dictionaryVersion = version;
+            updateDictionaryEntry();
+        }
     }
 
-
-    public void loadFromResourceFiles() {
-        dictionary.computeIfAbsent(NAME_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
+    private void updateDictionaryEntry() {
+        try {
+            TypeResponse typeResponse = dictionaryClient.getAllTypes();
+            if (typeResponse == null || CollectionUtils.isEmpty(typeResponse.getTypes())) {
+                return;
+            } else {
+                entryColors = typeResponse.getTypes().stream().collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
+                dictionaryEntry = entryColors.keySet().stream().collect(Collectors.toMap(type -> type, s -> dictionaryClient.getDictionaryForType(s).getEntries().stream().collect(Collectors.toSet())));
+            }
+        } catch (FeignException e) {
+            log.warn("Got some unknown feignException", e);
+            throw e;
+        }
     }
-
-
-    private String cleanDictionaryEntry(String entry) {
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-}
+}

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionService.java

@@ -19,15 +19,16 @@ import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 
 @Service
 @RequiredArgsConstructor
+@Slf4j
 public class EntityRedactionService {
 
     private final DictionaryService dictionaryService;
     private final DroolsExecutionService droolsExecutionService;
 
-
     public void processDocument(Document classifiedDoc) {
 
         dictionaryService.updateDictionary();
@@ -98,13 +99,12 @@ public class EntityRedactionService {
         });
     }
 
-
     private Set<Entity> findEntities(SearchableText searchableText, String headline) {
 
         String normalizedInputString = searchableText.toString();
 
         Set<Entity> found = new HashSet<>();
-        for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionary().entrySet()) {
+        for (Map.Entry<String, Set<String>> entry : dictionaryService.getDictionaryEntry().entrySet()) {
             for (String value : entry.getValue()) {
                 int startIndex;
                 int stopIndex = 0;
@@ -130,7 +130,6 @@ public class EntityRedactionService {
         return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
     }
 
-
     public void removeEntitiesContainedInLarger(Set<Entity> entities) {
         List<Entity> wordsToRemove = new ArrayList<>();
         for (Entity word : entities) {
@@ -142,6 +141,4 @@ public class EntityRedactionService {
         }
         entities.removeAll(wordsToRemove);
     }
-
-
 }

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/visualization/service/AnnotationHighlightService.java

@@ -1,10 +1,5 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;
 
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
-
 import java.awt.Color;
 import java.io.IOException;
 import java.util.List;
@@ -29,6 +24,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -40,7 +36,7 @@ import lombok.extern.slf4j.Slf4j;
 @Service
 @RequiredArgsConstructor
 public class AnnotationHighlightService {
-
+    private final DictionaryService dictionaryService;
 
     public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {
 
@@ -177,36 +173,16 @@ public class AnnotationHighlightService {
         if (!entity.isRedaction()) {
             return false;
         }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return true;
-        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return true;
-        }
-        return false;
+        return dictionaryService.getDictionaryEntry().keySet().contains(entity.getType());
     }
 
-
     private float[] getColor(Entity entity) {
         if (!entity.isRedaction()) {
             return new float[]{0.627f, 0.627f, 0.627f};
         }
-        if (entity.getType().equals(VERTEBRATES_CODE)) {
-            return new float[]{0, 1, 0};
-        }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return new float[]{0, 1, 1};
-        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return new float[]{1, 1, 0};
-        }
-        if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
-            return new float[]{1, 0.502f, 0};
-        }
-        return null;
+        return dictionaryService.getEntryColors().get(entity.getType());
     }
 
-
     private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
 
         contentStream.setStrokingColor(Color.LIGHT_GRAY);