Pull request #11 : Feature/RED-161: replace hard-coded hint-type with dictionary-service query

Merge in RED/redaction-service from feature/RED-161 to master * commit '3eb37fa6db0d9f1a4e6b93b2047b3b96c94cd9ff': RED-161: reverse caseSensitive RED-161: replace hard-coded hint-type with dictionary-service query
RED-161: reverse caseSensitive
2020-07-23 10:59:37 +02:00 · 2020-07-23 10:51:30 +02:00 · 2020-07-22 22:54:05 +02:00 · 2020-07-22 14:15:56 +02:00 · 2020-07-22 14:14:10 +02:00 · 2020-07-22 14:06:45 +02:00
17 changed files with 279 additions and 114 deletions
@@ -13,6 +13,7 @@ public class RedactionLogEntry {
    private String value;
    private String reason;
    private boolean redacted;
+    private boolean isHint;
    private String section;
    private float[] color;
    private List<Rectangle> positions = new ArrayList<>();
@@ -39,7 +39,7 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.6</version>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -45,7 +45,7 @@ public class Application {
        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
        RulesResponse rules = rulesClient.getRules();
        InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
-        kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
+        kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
        kieBuilder.buildAll();
        KieModule kieModule = kieBuilder.getKieModule();
@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface DictionaryClient extends DictionaryResource {
+}
@@ -49,15 +49,6 @@ public class Section {
        });
    }

-    public void highlightAll(String type){
-        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
-                entity.setRedaction(true);
-            }
-        });
-    }
-
-
    public void redactLineAfter(String start, String asType, int ruleNumber, String reason){

        String value = StringUtils.substringBetween(text, start, "\n");
@@ -1,58 +1,85 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

-import javax.annotation.PostConstruct;
-
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;

+import feign.FeignException;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
+@Slf4j
 public class DictionaryService {

-    public static final String VERTEBRATES_CODE = "VERTEBRATE";
-    public static final String ADDRESS_CODE = "ADDRESS";
-    public static final String NAME_CODE = "NAME";
-    public static final String NO_REDACTION_INDICATOR = "NO_REDACTION_INDICATOR";
+    private final DictionaryClient dictionaryClient;
+
+    private long dictionaryVersion = -1;

    @Getter
    private Map<String, Set<String>> dictionary = new HashMap<>();

    @Getter
-    private long generation;
+    private Map<String, float[]> entryColors = new HashMap<>();

-    @PostConstruct
-    public void init() {
-        loadFromResourceFiles();
-    }
+    @Getter
+    private List<String> hintTypes = new ArrayList<>();
+
+    @Getter
+    private List<String> caseInsensitiveTypes = new ArrayList<>();


    public void updateDictionary() {
-        //TODO
+
+        long version = dictionaryClient.getVersion();
+        if (version > dictionaryVersion) {
+            dictionaryVersion = version;
+            updateDictionaryEntry();
+        }
    }


-    public void loadFromResourceFiles() {
-        dictionary.computeIfAbsent(NAME_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
+    private void updateDictionaryEntry() {
+
+        try {
+            TypeResponse typeResponse = dictionaryClient.getAllTypes();
+            if (typeResponse != null && !CollectionUtils.isEmpty(typeResponse.getTypes())) {
+                entryColors = typeResponse.getTypes()
+                        .stream()
+                        .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
+                dictionary = entryColors.keySet()
+                        .stream()
+                        .collect(Collectors.toMap(type -> type, s -> dictionaryClient.getDictionaryForType(s)
+                                .getEntries()
+                                .stream()
+                                .collect(Collectors.toSet())));
+                hintTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isHint)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                caseInsensitiveTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isCaseInsensitive)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+            }
+        } catch (FeignException e) {
+            log.warn("Got some unknown feignException", e);
+            throw e;
+        }
    }

-
-    private String cleanDictionaryEntry(String entry) {
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-}
+}
@@ -62,7 +62,7 @@ public class DroolsExecutionService {
            KieServices kieServices = KieServices.Factory.get();
            InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
            kieBuilder.buildAll();
            KieModule kieModule = kieBuilder.getKieModule();
@@ -27,7 +27,6 @@ public class EntityRedactionService {
    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;

-
    public void processDocument(Document classifiedDoc) {

        dictionaryService.updateDictionary();
@@ -98,7 +97,6 @@ public class EntityRedactionService {
        });
    }

-
    private Set<Entity> findEntities(SearchableText searchableText, String headline) {

        String normalizedInputString = searchableText.toString();
@@ -130,7 +128,6 @@ public class EntityRedactionService {
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

-
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
@@ -142,6 +139,4 @@ public class EntityRedactionService {
        }
        entities.removeAll(wordsToRemove);
    }
-
-
 }
@@ -1,14 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;

-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
-
 import java.awt.Color;
 import java.io.IOException;
 import java.util.List;

+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -29,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -41,6 +38,8 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class AnnotationHighlightService {

+    private final DictionaryService dictionaryService;
+

    public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {

@@ -102,18 +101,29 @@ public class AnnotationHighlightService {
                            posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
                            posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
                            posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 4;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() - height + 4;
                        } else {

                            posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
-                            posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
-                            posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj() + 2;
+                            posXEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() + textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getWidth() + 1;
+                            posYInit = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getYDirAdj() + 2;
                        }

                        Rectangle textHighlightRectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);

-
                        List<PDAnnotation> annotations = pdPage.getAnnotations();
                        PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
                        highlight.constructAppearances();
@@ -125,10 +135,11 @@ public class AnnotationHighlightService {
                        annotationPosition.setUpperRightY(posYEnd + height);

                        highlight.setRectangle(annotationPosition);
-                        if (!flatRedaction) {
+                        if (!flatRedaction && !isHint(entity)) {
                            highlight.setAnnotationName(entityPositionSequence.getId().toString());
                            highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity.getHeadline() + "\"");
+                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity
+                                    .getHeadline() + "\"");
                        }

                        highlight.setQuadPoints(toQuadPoints(textHighlightRectangle));
@@ -154,6 +165,7 @@ public class AnnotationHighlightService {
                redactionLogEntry.setType(entity.getType());
                redactionLogEntry.setRedacted(entity.isRedaction());
                redactionLogEntry.setSection(entity.getHeadline());
+                redactionLogEntry.setHint(isHint(entity));
                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
            }

@@ -165,47 +177,45 @@ public class AnnotationHighlightService {

        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
        // of the area to be highlighted
-        return new float[]{
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
+        return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
+                .getY() + rectangle.getHeight(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
    }


    private boolean isRedactionType(Entity entity) {
+
        if (!entity.isRedaction()) {
            return false;
        }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return true;
+        if (isHint(entity)) {
+            return false;
        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return true;
-        }
-        return false;
+        return true;
    }


    private float[] getColor(Entity entity) {
-        if (!entity.isRedaction()) {
+
+        if (!entity.isRedaction() && !isHint(entity)) {
            return new float[]{0.627f, 0.627f, 0.627f};
        }
-        if (entity.getType().equals(VERTEBRATES_CODE)) {
-            return new float[]{0, 1, 0};
+
+        if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
+            return dictionaryService.getEntryColors().get("default");
        }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return new float[]{0, 1, 1};
-        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return new float[]{1, 1, 0};
-        }
-        if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
-            return new float[]{1, 0.502f, 0};
-        }
-        return null;
+
+        return dictionaryService.getEntryColors().get(entity.getType());
    }

+    private boolean isHint(Entity entity) {
+        List<String> hintTypes = dictionaryService.getHintTypes();
+        if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
+            return true;
+        }
+        return false;
+    }

    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {

@@ -231,13 +241,15 @@ public class AnnotationHighlightService {


    private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+
        for (List<Cell> row : table.getRows()) {
            for (Cell cell : row) {

                if (cell != null) {
                    contentStream.setLineWidth(0.5f);
                    contentStream.setStrokingColor(Color.CYAN);
-                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell.getHeight());
+                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
+                            .getHeight());
                    contentStream.stroke();

 //                    contentStream.setStrokingColor(Color.GREEN);
@@ -262,4 +274,5 @@ public class AnnotationHighlightService {
            contentStream.endText();
        }
    }
+
 }
@@ -11,6 +11,11 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
@@ -28,26 +33,43 @@ import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.ResourceLoader;
 import org.springframework.test.context.junit4.SpringRunner;

+import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

@Ignore
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {

+    public static final String VERTEBRATES_CODE = "vertebrate";
+    public static final String ADDRESS_CODE = "address";
+    public static final String NAME_CODE = "name";
+    public static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    public static final String DEFAULT = "default";
+
    @Autowired
    private RedactionController redactionController;

    @MockBean
    private RulesClient rulesClient;

+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    private Map<String, List<String>> dictionary = new HashMap<>();
+    private Map<String, float[]> typeColorMap = new HashMap<>();
+
    @TestConfiguration
    public static class RedactionIntegrationTestConfiguration {

@@ -64,9 +86,7 @@ public class RedactionIntegrationTest {
            KieModule kieModule = kieBuilder.getKieModule();

            return kieServices.newKieContainer(kieModule.getReleaseId());
-
        }
-
    }

    @Before
@@ -75,6 +95,43 @@ public class RedactionIntegrationTest {
        when(rulesClient.getVersion()).thenReturn(0L);
        when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));

+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion()).thenReturn(0L);
+        when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
+        when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(DEFAULT)).thenReturn(getDictionaryResponse(DEFAULT));
+    }
+
+    private void loadDictionaryForTest() {
+        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toSet()));
+        dictionary.put(DEFAULT, new ArrayList<>());
+    }
+
+    private String cleanDictionaryEntry(String entry) {
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+    private void loadTypeForTest() {
+        typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
+        typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
+        typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
+        typeColorMap.put(DEFAULT, new float[]{1, 0.502f, 0});
+    }
+
+    private List<TypeResult> getTypeResponse() {
+        return typeColorMap.entrySet().stream().map(typeColor -> TypeResult.builder().type(typeColor.getKey()).color(typeColor.getValue()).build()).collect(Collectors.toList());
+    }
+
+    private DictionaryResponse getDictionaryResponse(String type) {
+        return DictionaryResponse.builder().color(typeColorMap.get(type)).entries(dictionary.get(type)).build();
    }

    @Test
@@ -171,7 +228,5 @@ public class RedactionIntegrationTest {
        } catch (IOException e) {
            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
        }
-
    }
-
 }
@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class ResourceLoader {
+
+    public Map<String, String> loadDictionaryFiles() {
+
+        String name = "dictionaries/";
+
+        List<String> files;
+        try {
+            files = IOUtils.readLines(ResourceLoader.class.getClassLoader().getResourceAsStream(name), "UTF-8");
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + name, e);
+        }
+        return files.stream().collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s));
+    }
+
+    private String getFileName(String filePath) {
+        return filePath.substring(0, filePath.indexOf(".txt"));
+    }
+
+    public Set<String> load(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.toSet());
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+    }
+
+    public String loadToString(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.joining("\n"));
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+
+    }
+
+}
@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TextNormalizationUtilities {
+
+    /**
+     * Revert hyphenation due to line breaks.
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+    }
+
+}
@@ -4,39 +4,31 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Section

 global Section section

-rule "0: Highlight Indicators"
-	when
-		eval(section.getEntities().isEmpty()==false);
-	then
-		section.highlightAll("VERTEBRATE");
-		section.highlightAll("NO_REDACTION_INDICATOR");
-	end
-

 rule "1: Redacted because Section contains Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==true);
+		eval(section.contains("vertebrate")==true);
 	then
-		section.redact("NAME", 1, "Redacted because Section contains Vertebrate");
-		section.redact("ADDRESS", 1, "Redacted because Section contains Vertebrate");
+		section.redact("name", 1, "Redacted because Section contains Vertebrate");
+		section.redact("address", 1, "Redacted because Section contains Vertebrate");
 	end


 rule "2: Not Redacted because Section contains no Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==false);
+		eval(section.contains("vertebrate")==false);
 	then
-		section.redactNot("NAME", 2, "Not Redacted because Section contains no Vertebrate");
-		section.redactNot("ADDRESS", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate");
 	end


 rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
 	when
-		eval(section.contains("VERTEBRATE")==true && section.contains("NO_REDACTION_INDICATOR")==true);
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true);
 	then
-		section.redactNot("NAME", 3, "Vertebrate was found, but also a no redaction indicator");
-		section.redactNot("ADDRESS", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator");
 	end


@@ -44,12 +36,12 @@ rule "4: Redact contact information, if applicant is found"
 	when
 		eval(section.getText().toLowerCase().contains("applicant"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Contact point:",  "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "ADDRESS", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4");
+		section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Contact point:",  "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4");
 	end


@@ -57,10 +49,10 @@ rule "5: Redact contact information, if 'Producer of the plant protection produc
 	when
 		eval(section.getText().contains("Producer of the plant protection product"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Contact:", "Phone", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Phone:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Fax:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("E-mail:", "ADDRESS", 5, "xxxx");
+		section.redactLineAfter("Name:", "address", 5, "xxxx");
+		section.redactBetween("Address:", "Contact", "address", 5, "xxxx");
+		section.redactBetween("Contact:", "Phone", "address", 5, "xxxx");
+		section.redactLineAfter("Phone:", "address", 5, "xxxx");
+		section.redactLineAfter("Fax:", "address", 5, "xxxx");
+		section.redactLineAfter("E-mail:", "address", 5, "xxxx");
 	end
Author	SHA1	Message	Date
Thierry Goeckel	9a425a8594	Pull request #11 : Feature/RED-161: replace hard-coded hint-type with dictionary-service query Merge in RED/redaction-service from feature/RED-161 to master * commit '3eb37fa6db0d9f1a4e6b93b2047b3b96c94cd9ff': RED-161: reverse caseSensitive RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-23 10:59:37 +02:00
cheng	3eb37fa6db	RED-161: reverse caseSensitive	2020-07-23 10:51:30 +02:00
cheng	c8375a1ad9	RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-22 22:54:05 +02:00
Dominique Eiflaender	94827a417f	Pull request #10 : Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false Merge in RED/redaction-service from RedactionLog1 to master * commit 'e7ea12a1b03f7f79e0f4fa12671f91ee326eef73': CodeStyle Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:15:56 +02:00
deiflaender	e7ea12a1b0	CodeStyle	2020-07-22 14:14:10 +02:00
deiflaender	6a9df397eb	Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:06:45 +02:00
Cheng Zhu	41c61dd214	Pull request #9 : DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming. Merge in RED/redaction-service from Fix1 to master * commit '49fe418a60f424e373038b1f7b9d43ff4afde37c': DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:20:22 +02:00
deiflaender	49fe418a60	DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:17:52 +02:00
Dominique Eiflaender	c98f1d5e37	Pull request #8 : RED-169: Fixed startup problem Merge in RED/redaction-service from RED-169 to master * commit 'c0882346a9d1554c51ccc526d1f6d7c277ffbc7c': RED-169: Fixed startup problem	2020-07-21 14:53:55 +02:00
deiflaender	c0882346a9	RED-169: Fixed startup problem	2020-07-21 14:49:00 +02:00
Dominique Eiflaender	302daff526	Pull request #6 : RED-106: replace the local dictionary preload with remove dictionary service. Merge in RED/redaction-service from feature/RED-106i to master * commit 'd607ac567d1e07cfc4c7dd4fa581d1881a874537': RED-106: integrationTest fixed. RED-106: integrationTest fixed. RED-106: integrationTest fixed. REd-106: rebase auf master REd-106: rebase auf master DEV: bugfix missing bean REd-106: enable dictionary version RED-106: replace the local dictionary preload with remove dictionary service. RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-21 13:19:27 +02:00
cheng	d607ac567d	RED-106: integrationTest fixed.	2020-07-21 13:15:58 +02:00
cheng	1b40931c87	RED-106: integrationTest fixed.	2020-07-21 13:14:17 +02:00
cheng	75c6433eef	RED-106: integrationTest fixed.	2020-07-21 13:14:10 +02:00
cheng	9c14f331fd	REd-106: rebase auf master	2020-07-20 10:13:35 +02:00
cheng	141e4b9371	REd-106: rebase auf master	2020-07-20 09:19:54 +02:00
cheng	902d5cddf7	DEV: bugfix missing bean	2020-07-20 09:18:42 +02:00
cheng	b3841b86b1	REd-106: enable dictionary version	2020-07-20 09:18:42 +02:00
cheng	4e3a6af3f8	RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-20 09:18:42 +02:00
cheng	8ed548f1a8	RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-20 09:18:42 +02:00
Thierry Goeckel	fe5c20e1a0	Pull request #7 : Fix application startup and remove redundant code. Merge in RED/redaction-service from bugfix/application-startup to master * commit '3e18fe19f31081df98670130512ab3933f7015af': Fix application startup and remove redundant code.	2020-07-20 09:18:21 +02:00