Pull request #12 : Bugfix/RED-183

Merge in RED/redaction-service from bugfix/RED-183 to master * commit '7dbe03483b7a8a3c391e9d0791717b9fab5fc5db': RED-183: Fix catching validation errors Fix style.
RED-183: Fix catching validation errors
2020-07-23 14:48:11 +02:00 · 2020-07-23 13:36:02 +02:00 · 2020-07-23 13:35:58 +02:00 · 2020-07-23 10:59:37 +02:00 · 2020-07-23 10:51:30 +02:00 · 2020-07-22 22:54:05 +02:00
17 changed files with 391 additions and 161 deletions
@@ -13,6 +13,7 @@ public class RedactionLogEntry {
    private String value;
    private String reason;
    private boolean redacted;
+    private boolean isHint;
    private String section;
    private float[] color;
    private List<Rectangle> positions = new ArrayList<>();
@@ -39,7 +39,7 @@
        <dependency>
            <groupId>com.iqser.red.service</groupId>
            <artifactId>configuration-service-api-v1</artifactId>
-            <version>1.0.0</version>
+            <version>1.0.6</version>
        </dependency>
        <dependency>
            <groupId>org.drools</groupId>
@@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;

+import org.apache.commons.lang3.StringUtils;
 import org.kie.api.KieServices;
 import org.kie.api.builder.KieBuilder;
 import org.kie.api.builder.KieFileSystem;
@@ -22,6 +23,7 @@ import org.springframework.context.annotation.Import;
 import com.iqser.gin4.commons.spring.DefaultWebMvcConfiguration;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
+import com.iqser.red.service.redaction.v1.server.exception.RulesValidationException;
 import com.iqser.red.service.redaction.v1.server.settings.RedactionServiceSettings;

@Import({DefaultWebMvcConfiguration.class})
@@ -33,24 +35,35 @@ public class Application {
    @Autowired
    private RulesClient rulesClient;

+
    public static void main(String[] args) {
+
        SpringApplication.run(Application.class, args);
    }

+
    @Bean
    public KieContainer kieContainer() {

-        KieServices kieServices = KieServices.Factory.get();
+        try {
+            KieServices kieServices = KieServices.Factory.get();

-        KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-        RulesResponse rules = rulesClient.getRules();
-        InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
-        kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
-        KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
-        kieBuilder.buildAll();
-        KieModule kieModule = kieBuilder.getKieModule();
+            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
+            RulesResponse rules = rulesClient.getRules();
+            if (StringUtils.isEmpty(rules.getRules())) {
+                throw new RuntimeException("Rules cannot be empty.");
+            }
+            InputStream input = new ByteArrayInputStream(rules.getRules().getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources()
+                    .newInputStreamResource(input));
+            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
+            kieBuilder.buildAll();
+            KieModule kieModule = kieBuilder.getKieModule();

-        return kieServices.newKieContainer(kieModule.getReleaseId());
+            return kieServices.newKieContainer(kieModule.getReleaseId());
+        } catch (Exception e) {
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
+        }

    }

@@ -0,0 +1,10 @@
+package com.iqser.red.service.redaction.v1.server.client;
+
+import org.springframework.cloud.openfeign.FeignClient;
+
+import com.iqser.red.service.configuration.v1.api.resource.DictionaryResource;
+import com.iqser.red.service.configuration.v1.api.resource.RulesResource;
+
+@FeignClient(name = "DictionaryResource", url = "http://" + RulesResource.SERVICE_NAME + ":8080")
+public interface DictionaryClient extends DictionaryResource {
+}
@@ -25,13 +25,17 @@ public class Section {

    private String headline;

+
    public boolean contains(String type) {
+
        return entities.stream().anyMatch(entity -> entity.getType().equals(type));
    }

-    public void redact(String type, int ruleNumber, String reason){
+
+    public void redact(String type, int ruleNumber, String reason) {
+
        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -39,9 +43,11 @@ public class Section {
        });
    }

-    public void redactNot(String type, int ruleNumber, String reason){
+
+    public void redactNot(String type, int ruleNumber, String reason) {
+
        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
+            if (entity.getType().equals(type)) {
                entity.setRedaction(false);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -49,27 +55,19 @@ public class Section {
        });
    }

-    public void highlightAll(String type){
-        entities.forEach(entity -> {
-            if(entity.getType().equals(type)){
-                entity.setRedaction(true);
-            }
-        });
-    }

-
-    public void redactLineAfter(String start, String asType, int ruleNumber, String reason){
+    public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {

        String value = StringUtils.substringBetween(text, start, "\n");

-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
            entities.addAll(found);
        }

        // TODO No need to iterate
        entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -79,19 +77,18 @@ public class Section {
    }


-
-    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason){
+    public void redactBetween(String start, String stop, String asType, int ruleNumber, String reason) {

        String value = StringUtils.substringBetween(searchText, start, stop);

-        if(value != null){
-            Set<Entity> found  = findEntity(value.trim(), asType);
+        if (value != null) {
+            Set<Entity> found = findEntity(value.trim(), asType);
            entities.addAll(found);
        }

        // TODO No need to iterate
        entities.forEach(entity -> {
-            if(entity.getType().equals(asType)){
+            if (entity.getType().equals(asType)) {
                entity.setRedaction(true);
                entity.setMatchedRule(ruleNumber);
                entity.setRedactionReason(reason);
@@ -100,25 +97,21 @@ public class Section {
    }


-
-
    private Set<Entity> findEntity(String value, String asType) {

        Set<Entity> found = new HashSet<>();

-                int startIndex;
-                int stopIndex = 0;
-                do {
-                    startIndex = searchText.indexOf(value, stopIndex);
-                    stopIndex = startIndex + value.length();
-
-                    if (startIndex > -1 &&
-                            (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText.charAt(startIndex - 1))) &&
-                            (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
-                        found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
-                    }
-                } while (startIndex > -1);
+        int startIndex;
+        int stopIndex = 0;
+        do {
+            startIndex = searchText.indexOf(value, stopIndex);
+            stopIndex = startIndex + value.length();

+            if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText
+                    .charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
+                found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline));
+            }
+        } while (startIndex > -1);

        removeEntitiesContainedInLarger(found);

@@ -127,14 +120,18 @@ public class Section {


    private boolean isSeparator(char c) {
+
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

+
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
+
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
            for (Entity inner : entities) {
-                if (inner.getWord().length() < word.getWord().length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
+                if (inner.getWord().length() < word.getWord()
+                        .length() && inner.getStart() >= word.getStart() && inner.getEnd() <= word.getEnd() && word != inner) {
                    wordsToRemove.add(inner);
                }
            }
@@ -1,58 +1,84 @@
 package com.iqser.red.service.redaction.v1.server.redaction.service;

+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;

-import javax.annotation.PostConstruct;
-
+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Service;

-import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
-import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;

+import feign.FeignException;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;

-@Slf4j
@Service
@RequiredArgsConstructor
+@Slf4j
 public class DictionaryService {

-    public static final String VERTEBRATES_CODE = "VERTEBRATE";
-    public static final String ADDRESS_CODE = "ADDRESS";
-    public static final String NAME_CODE = "NAME";
-    public static final String NO_REDACTION_INDICATOR = "NO_REDACTION_INDICATOR";
+    private final DictionaryClient dictionaryClient;
+
+    private long dictionaryVersion = -1;

    @Getter
    private Map<String, Set<String>> dictionary = new HashMap<>();

    @Getter
-    private long generation;
+    private Map<String, float[]> entryColors = new HashMap<>();

-    @PostConstruct
-    public void init() {
-        loadFromResourceFiles();
-    }
+    @Getter
+    private List<String> hintTypes = new ArrayList<>();
+
+    @Getter
+    private List<String> caseInsensitiveTypes = new ArrayList<>();


    public void updateDictionary() {
-        //TODO
+
+        long version = dictionaryClient.getVersion();
+        if (version > dictionaryVersion) {
+            dictionaryVersion = version;
+            updateDictionaryEntry();
+        }
    }


-    public void loadFromResourceFiles() {
-        dictionary.computeIfAbsent(NAME_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/names.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/vertebrates.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/addresses.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
-        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new HashSet<>()).addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt").stream().map(this::cleanDictionaryEntry).collect(Collectors.toList()));
+    private void updateDictionaryEntry() {
+
+        try {
+            TypeResponse typeResponse = dictionaryClient.getAllTypes();
+            if (typeResponse != null && CollectionUtils.isNotEmpty(typeResponse.getTypes())) {
+                entryColors = typeResponse.getTypes()
+                        .stream()
+                        .collect(Collectors.toMap(TypeResult::getType, TypeResult::getColor));
+                dictionary = entryColors.keySet()
+                        .stream()
+                        .collect(Collectors.toMap(type -> type, s -> new HashSet<>(dictionaryClient.getDictionaryForType(s)
+                                .getEntries())));
+                hintTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isHint)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+                caseInsensitiveTypes = typeResponse.getTypes()
+                        .stream()
+                        .filter(TypeResult::isCaseInsensitive)
+                        .map(TypeResult::getType)
+                        .collect(Collectors.toList());
+            }
+        } catch (FeignException e) {
+            log.warn("Got some unknown feignException", e);
+            throw e;
+        }
    }

-
-    private String cleanDictionaryEntry(String entry) {
-        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
-    }
-}
+}
@@ -62,13 +62,13 @@ public class DroolsExecutionService {
            KieServices kieServices = KieServices.Factory.get();
            InputStream input = new ByteArrayInputStream(drlAsString.getBytes(StandardCharsets.UTF_8));
            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            kieFileSystem.write(kieServices.getResources().newInputStreamResource(input));
+            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
            kieBuilder.buildAll();
            KieModule kieModule = kieBuilder.getKieModule();
            kieContainer.updateToVersion(kieModule.getReleaseId());
        } catch (Exception e) {
-            throw new RulesValidationException("Could not update rules", e);
+            throw new RulesValidationException("Could not update rules: " + e.getMessage(), e);
        }

    }
@@ -27,7 +27,6 @@ public class EntityRedactionService {
    private final DictionaryService dictionaryService;
    private final DroolsExecutionService droolsExecutionService;

-
    public void processDocument(Document classifiedDoc) {

        dictionaryService.updateDictionary();
@@ -98,7 +97,6 @@ public class EntityRedactionService {
        });
    }

-
    private Set<Entity> findEntities(SearchableText searchableText, String headline) {

        String normalizedInputString = searchableText.toString();
@@ -130,7 +128,6 @@ public class EntityRedactionService {
        return Character.isWhitespace(c) || Pattern.matches("\\p{Punct}", String.valueOf(c)) || c == '\"' || c == '‘' || c == '’';
    }

-
    public void removeEntitiesContainedInLarger(Set<Entity> entities) {
        List<Entity> wordsToRemove = new ArrayList<>();
        for (Entity word : entities) {
@@ -142,6 +139,4 @@ public class EntityRedactionService {
        }
        entities.removeAll(wordsToRemove);
    }
-
-
 }
@@ -1,14 +1,10 @@
 package com.iqser.red.service.redaction.v1.server.visualization.service;

-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.ADDRESS_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NAME_CODE;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.NO_REDACTION_INDICATOR;
-import static com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService.VERTEBRATES_CODE;
-
 import java.awt.Color;
 import java.io.IOException;
 import java.util.List;

+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.pdmodel.PDPage;
 import org.apache.pdfbox.pdmodel.PDPageContentStream;
@@ -29,6 +25,7 @@ import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
 import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
 import com.iqser.red.service.redaction.v1.server.redaction.model.Entity;
 import com.iqser.red.service.redaction.v1.server.redaction.model.EntityPositionSequence;
+import com.iqser.red.service.redaction.v1.server.redaction.service.DictionaryService;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
 import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
@@ -41,6 +38,8 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
 public class AnnotationHighlightService {

+    private final DictionaryService dictionaryService;
+

    public void highlight(PDDocument document, Document classifiedDoc, boolean flatRedaction) throws IOException {

@@ -102,18 +101,29 @@ public class AnnotationHighlightService {
                            posXEnd = textPositions.getTextPositions().get(0).getYDirAdj() + 2;
                            posXInit = textPositions.getTextPositions().get(0).getYDirAdj() - height;
                            posYInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posYEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() - height + 4;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() - height + 4;
                        } else {

                            posXInit = textPositions.getTextPositions().get(0).getXDirAdj();
-                            posXEnd = textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getXDirAdj() + textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getWidth() + 1;
-                            posYInit = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
-                            posYEnd = textPositions.getTextPositions().get(0).getPageHeight() - textPositions.getTextPositions().get(textPositions.getTextPositions().size() - 1).getYDirAdj() + 2;
+                            posXEnd = textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getXDirAdj() + textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getWidth() + 1;
+                            posYInit = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions().get(0).getYDirAdj() - 2;
+                            posYEnd = textPositions.getTextPositions()
+                                    .get(0)
+                                    .getPageHeight() - textPositions.getTextPositions()
+                                    .get(textPositions.getTextPositions().size() - 1)
+                                    .getYDirAdj() + 2;
                        }

                        Rectangle textHighlightRectangle = new Rectangle(new Point(posXInit, posYInit), posXEnd - posXInit, posYEnd - posYInit + height, page);

-
                        List<PDAnnotation> annotations = pdPage.getAnnotations();
                        PDAnnotationTextMarkup highlight = new PDAnnotationTextMarkup(PDAnnotationTextMarkup.SUB_TYPE_HIGHLIGHT);
                        highlight.constructAppearances();
@@ -125,10 +135,11 @@ public class AnnotationHighlightService {
                        annotationPosition.setUpperRightY(posYEnd + height);

                        highlight.setRectangle(annotationPosition);
-                        if (!flatRedaction) {
+                        if (!flatRedaction && !isHint(entity)) {
                            highlight.setAnnotationName(entityPositionSequence.getId().toString());
                            highlight.setTitlePopup(entityPositionSequence.getId().toString());
-                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity.getHeadline() + "\"");
+                            highlight.setContents("\nRule " + entity.getMatchedRule() + " matched\n\n" + entity.getRedactionReason() + "\n\n" + "In Section : \"" + entity
+                                    .getHeadline() + "\"");
                        }

                        highlight.setQuadPoints(toQuadPoints(textHighlightRectangle));
@@ -154,6 +165,7 @@ public class AnnotationHighlightService {
                redactionLogEntry.setType(entity.getType());
                redactionLogEntry.setRedacted(entity.isRedaction());
                redactionLogEntry.setSection(entity.getHeadline());
+                redactionLogEntry.setHint(isHint(entity));
                classifiedDoc.getRedactionLogEntities().add(redactionLogEntry);
            }

@@ -165,47 +177,45 @@ public class AnnotationHighlightService {

        // quadPoints is array of x,y coordinates in Z-like order (top-left, top-right, bottom-left,bottom-right)
        // of the area to be highlighted
-        return new float[]{
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(),
-                rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY() + rectangle.getHeight(),
-                rectangle.getTopLeft().getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
+        return new float[]{rectangle.getTopLeft().getX(), rectangle.getTopLeft().getY(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY(), rectangle.getTopLeft().getX(), rectangle.getTopLeft()
+                .getY() + rectangle.getHeight(), rectangle.getTopLeft()
+                .getX() + rectangle.getWidth(), rectangle.getTopLeft().getY() + rectangle.getHeight()};
    }


    private boolean isRedactionType(Entity entity) {
+
        if (!entity.isRedaction()) {
            return false;
        }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return true;
+        if (isHint(entity)) {
+            return false;
        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return true;
-        }
-        return false;
+        return true;
    }


    private float[] getColor(Entity entity) {
-        if (!entity.isRedaction()) {
+
+        if (!entity.isRedaction() && !isHint(entity)) {
            return new float[]{0.627f, 0.627f, 0.627f};
        }
-        if (entity.getType().equals(VERTEBRATES_CODE)) {
-            return new float[]{0, 1, 0};
+
+        if (!dictionaryService.getEntryColors().containsKey(entity.getType())) {
+            return dictionaryService.getEntryColors().get("default");
        }
-        if (entity.getType().equals(ADDRESS_CODE)) {
-            return new float[]{0, 1, 1};
-        }
-        if (entity.getType().equals(NAME_CODE)) {
-            return new float[]{1, 1, 0};
-        }
-        if (entity.getType().equals(NO_REDACTION_INDICATOR)) {
-            return new float[]{1, 0.502f, 0};
-        }
-        return null;
+
+        return dictionaryService.getEntryColors().get(entity.getType());
    }

+    private boolean isHint(Entity entity) {
+        List<String> hintTypes = dictionaryService.getHintTypes();
+        if (CollectionUtils.isNotEmpty(hintTypes) && hintTypes.contains(entity.getType())) {
+            return true;
+        }
+        return false;
+    }

    private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {

@@ -231,13 +241,15 @@ public class AnnotationHighlightService {


    private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
+
        for (List<Cell> row : table.getRows()) {
            for (Cell cell : row) {

                if (cell != null) {
                    contentStream.setLineWidth(0.5f);
                    contentStream.setStrokingColor(Color.CYAN);
-                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell.getHeight());
+                    contentStream.addRect((float) cell.getX(), (float) cell.getY(), (float) cell.getWidth(), (float) cell
+                            .getHeight());
                    contentStream.stroke();

 //                    contentStream.setStrokingColor(Color.GREEN);
@@ -262,4 +274,5 @@ public class AnnotationHighlightService {
            contentStream.endText();
        }
    }
+
 }
@@ -11,6 +11,11 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;

 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
@@ -28,26 +33,44 @@ import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.boot.test.mock.mockito.MockBean;
 import org.springframework.context.annotation.Bean;
 import org.springframework.core.io.ClassPathResource;
-import org.springframework.core.io.ResourceLoader;
 import org.springframework.test.context.junit4.SpringRunner;

+import com.iqser.red.service.configuration.v1.api.model.DictionaryResponse;
 import com.iqser.red.service.configuration.v1.api.model.RulesResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResponse;
+import com.iqser.red.service.configuration.v1.api.model.TypeResult;
 import com.iqser.red.service.redaction.v1.model.RedactionRequest;
 import com.iqser.red.service.redaction.v1.model.RedactionResult;
+import com.iqser.red.service.redaction.v1.server.client.DictionaryClient;
 import com.iqser.red.service.redaction.v1.server.client.RulesClient;
 import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
+import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;

@Ignore
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {

+    private static final String RULES = loadFromClassPath("drools/rules.drl");
+    private static final String VERTEBRATES_CODE = "vertebrate";
+    private static final String ADDRESS_CODE = "address";
+    private static final String NAME_CODE = "name";
+    private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String DEFAULT = "default";
+
    @Autowired
    private RedactionController redactionController;

    @MockBean
    private RulesClient rulesClient;

+    @MockBean
+    private DictionaryClient dictionaryClient;
+
+    private final Map<String, List<String>> dictionary = new HashMap<>();
+    private final Map<String, float[]> typeColorMap = new HashMap<>();
+
    @TestConfiguration
    public static class RedactionIntegrationTestConfiguration {

@@ -57,33 +80,102 @@ public class RedactionIntegrationTest {
            KieServices kieServices = KieServices.Factory.get();

            KieFileSystem kieFileSystem = kieServices.newKieFileSystem();
-            InputStream input = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
-            kieFileSystem.write("src/main/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
+            InputStream input = new ByteArrayInputStream(RULES.getBytes(StandardCharsets.UTF_8));
+            kieFileSystem.write("src/test/resources/drools/rules.drl", kieServices.getResources().newInputStreamResource(input));
            KieBuilder kieBuilder = kieServices.newKieBuilder(kieFileSystem);
            kieBuilder.buildAll();
            KieModule kieModule = kieBuilder.getKieModule();

            return kieServices.newKieContainer(kieModule.getReleaseId());
-
        }

    }

+
    @Before
    public void stubRulesClient() {

        when(rulesClient.getVersion()).thenReturn(0L);
-        when(rulesClient.getRules()).thenReturn(new RulesResponse(loadFromClassPath("drools/rules.drl")));
+        when(rulesClient.getRules()).thenReturn(new RulesResponse(RULES));

+        loadDictionaryForTest();
+        loadTypeForTest();
+        when(dictionaryClient.getVersion()).thenReturn(0L);
+        when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
+        when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
+        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
+        when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(DEFAULT)).thenReturn(getDictionaryResponse(DEFAULT));
    }

+
+    private void loadDictionaryForTest() {
+
+        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/names.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/addresses.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.put(DEFAULT, new ArrayList<>());
+    }
+
+
+    private String cleanDictionaryEntry(String entry) {
+
+        return TextNormalizationUtilities.removeHyphenLineBreaks(entry).replaceAll("\\n", " ");
+    }
+
+
+    private void loadTypeForTest() {
+
+        typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
+        typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
+        typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
+        typeColorMap.put(DEFAULT, new float[]{1, 0.502f, 0});
+    }
+
+
+    private List<TypeResult> getTypeResponse() {
+
+        return typeColorMap.entrySet()
+                .stream()
+                .map(typeColor -> TypeResult.builder().type(typeColor.getKey()).color(typeColor.getValue()).build())
+                .collect(Collectors.toList());
+    }
+
+
+    private DictionaryResponse getDictionaryResponse(String type) {
+
+        return DictionaryResponse.builder().color(typeColorMap.get(type)).entries(dictionary.get(type)).build();
+    }
+
+
    @Test
    public void redactionTest() throws IOException {

        long start = System.currentTimeMillis();
        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();
        request.setFlatRedaction(false);

        RedactionResult result = redactionController.redact(request);
@@ -97,12 +189,15 @@ public class RedactionIntegrationTest {
        System.out.println("numberOfPages: " + result.getNumberOfPages());
    }

+
    @Test
    public void classificationTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.classify(request);

@@ -111,12 +206,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void sectionsTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.sections(request);

@@ -125,12 +223,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void htmlTablesTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Fludioxonil/51 Fludioxonil_RAR_02_Volume_2_2018-02-21.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.htmlTables(request);

@@ -139,12 +240,15 @@ public class RedactionIntegrationTest {
        }
    }

+
    @Test
    public void htmlTableRotationTest() throws IOException {

        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");

-        RedactionRequest request = RedactionRequest.builder().document(IOUtils.toByteArray(pdfFileResource.getInputStream())).build();
+        RedactionRequest request = RedactionRequest.builder()
+                .document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
+                .build();

        RedactionResult result = redactionController.htmlTables(request);

@@ -153,10 +257,10 @@ public class RedactionIntegrationTest {
        }
    }

-    private String loadFromClassPath(String path) {

-        URL resource = ResourceLoader.class.getClassLoader()
-                .getResource(path);
+    private static String loadFromClassPath(String path) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(path);
        if (resource == null) {
            throw new IllegalArgumentException("could not load classpath resource: drools/rules.drl");
        }
@@ -164,14 +268,12 @@ public class RedactionIntegrationTest {
            StringBuilder sb = new StringBuilder();
            String str;
            while ((str = br.readLine()) != null) {
-                sb.append(str)
-                        .append("\n");
+                sb.append(str).append("\n");
            }
            return sb.toString();
        } catch (IOException e) {
            throw new IllegalArgumentException("could not load classpath resource: " + path, e);
        }
-
    }

-}
+}
@@ -0,0 +1,64 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class ResourceLoader {
+
+    public Map<String, String> loadDictionaryFiles() {
+
+        String name = "dictionaries/";
+
+        List<String> files;
+        try {
+            files = IOUtils.readLines(ResourceLoader.class.getClassLoader().getResourceAsStream(name), "UTF-8");
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + name, e);
+        }
+        return files.stream().collect(Collectors.toMap(ResourceLoader::getFileName, s -> name + s));
+    }
+
+    private String getFileName(String filePath) {
+        return filePath.substring(0, filePath.indexOf(".txt"));
+    }
+
+    public Set<String> load(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.toSet());
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+    }
+
+    public String loadToString(String classpathPath) {
+
+        URL resource = ResourceLoader.class.getClassLoader().getResource(classpathPath);
+        if (resource == null) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath);
+        }
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), StandardCharsets.UTF_8))) {
+            return br.lines().collect(Collectors.joining("\n"));
+        } catch (IOException e) {
+            throw new IllegalArgumentException("could not load classpath resource: " + classpathPath, e);
+        }
+
+    }
+
+}
@@ -0,0 +1,17 @@
+package com.iqser.red.service.redaction.v1.server.redaction.utils;
+
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
+public class TextNormalizationUtilities {
+
+    /**
+     * Revert hyphenation due to line breaks.
+     * @param text Text to be processed.
+     * @return Text without line-break hyphenation.
+     */
+    public static String removeHyphenLineBreaks(String text) {
+        return text.replaceAll("\\s(\\S+)[\\-\\u00AD]\\R|\n\r(.+ )", "\n$1$2");
+    }
+
+}
@@ -4,39 +4,31 @@ import com.iqser.red.service.redaction.v1.server.redaction.model.Section

 global Section section

-rule "0: Highlight Indicators"
-	when
-		eval(section.getEntities().isEmpty()==false);
-	then
-		section.highlightAll("VERTEBRATE");
-		section.highlightAll("NO_REDACTION_INDICATOR");
-	end
-

 rule "1: Redacted because Section contains Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==true);
+		eval(section.contains("vertebrate")==true);
 	then
-		section.redact("NAME", 1, "Redacted because Section contains Vertebrate");
-		section.redact("ADDRESS", 1, "Redacted because Section contains Vertebrate");
+		section.redact("name", 1, "Redacted because Section contains Vertebrate");
+		section.redact("address", 1, "Redacted because Section contains Vertebrate");
 	end


 rule "2: Not Redacted because Section contains no Vertebrate"
 	when
-		eval(section.contains("VERTEBRATE")==false);
+		eval(section.contains("vertebrate")==false);
 	then
-		section.redactNot("NAME", 2, "Not Redacted because Section contains no Vertebrate");
-		section.redactNot("ADDRESS", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate");
+		section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate");
 	end


 rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
 	when
-		eval(section.contains("VERTEBRATE")==true && section.contains("NO_REDACTION_INDICATOR")==true);
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true);
 	then
-		section.redactNot("NAME", 3, "Vertebrate was found, but also a no redaction indicator");
-		section.redactNot("ADDRESS", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator");
+		section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator");
 	end


@@ -44,12 +36,12 @@ rule "4: Redact contact information, if applicant is found"
 	when
 		eval(section.getText().toLowerCase().contains("applicant"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Contact point:",  "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "ADDRESS", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "ADDRESS", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4");
+		section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Contact point:",  "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4");
+		section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4");
 	end


@@ -57,10 +49,10 @@ rule "5: Redact contact information, if 'Producer of the plant protection produc
 	when
 		eval(section.getText().contains("Producer of the plant protection product"));
 	then
-		section.redactLineAfter("Name:", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Address:", "Contact", "ADDRESS", 5, "xxxx");
-		section.redactBetween("Contact:", "Phone", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Phone:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("Fax:", "ADDRESS", 5, "xxxx");
-		section.redactLineAfter("E-mail:", "ADDRESS", 5, "xxxx");
+		section.redactLineAfter("Name:", "address", 5, "xxxx");
+		section.redactBetween("Address:", "Contact", "address", 5, "xxxx");
+		section.redactBetween("Contact:", "Phone", "address", 5, "xxxx");
+		section.redactLineAfter("Phone:", "address", 5, "xxxx");
+		section.redactLineAfter("Fax:", "address", 5, "xxxx");
+		section.redactLineAfter("E-mail:", "address", 5, "xxxx");
 	end
Author	SHA1	Message	Date
Thierry Goeckel	ca439d821d	Pull request #12 : Bugfix/RED-183 Merge in RED/redaction-service from bugfix/RED-183 to master * commit '7dbe03483b7a8a3c391e9d0791717b9fab5fc5db': RED-183: Fix catching validation errors Fix style.	2020-07-23 14:48:11 +02:00
Thierry Göckel	7dbe03483b	RED-183: Fix catching validation errors	2020-07-23 13:36:02 +02:00
Thierry Göckel	33ab09d7fc	Fix style.	2020-07-23 13:35:58 +02:00
Thierry Goeckel	9a425a8594	Pull request #11 : Feature/RED-161: replace hard-coded hint-type with dictionary-service query Merge in RED/redaction-service from feature/RED-161 to master * commit '3eb37fa6db0d9f1a4e6b93b2047b3b96c94cd9ff': RED-161: reverse caseSensitive RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-23 10:59:37 +02:00
cheng	3eb37fa6db	RED-161: reverse caseSensitive	2020-07-23 10:51:30 +02:00
cheng	c8375a1ad9	RED-161: replace hard-coded hint-type with dictionary-service query	2020-07-22 22:54:05 +02:00
Dominique Eiflaender	94827a417f	Pull request #10 : Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false Merge in RED/redaction-service from RedactionLog1 to master * commit 'e7ea12a1b03f7f79e0f4fa12671f91ee326eef73': CodeStyle Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:15:56 +02:00
deiflaender	e7ea12a1b0	CodeStyle	2020-07-22 14:14:10 +02:00
deiflaender	6a9df397eb	Do not return redacted=true for vertebrates in RedactionLog, return isHint=true and redacted=false	2020-07-22 14:06:45 +02:00
Cheng Zhu	41c61dd214	Pull request #9 : DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming. Merge in RED/redaction-service from Fix1 to master * commit '49fe418a60f424e373038b1f7b9d43ff4afde37c': DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:20:22 +02:00
deiflaender	49fe418a60	DEV: Fixed Nullpointer on undefined color for type, adapted rule to Type naming.	2020-07-21 16:17:52 +02:00
Dominique Eiflaender	c98f1d5e37	Pull request #8 : RED-169: Fixed startup problem Merge in RED/redaction-service from RED-169 to master * commit 'c0882346a9d1554c51ccc526d1f6d7c277ffbc7c': RED-169: Fixed startup problem	2020-07-21 14:53:55 +02:00
deiflaender	c0882346a9	RED-169: Fixed startup problem	2020-07-21 14:49:00 +02:00
Dominique Eiflaender	302daff526	Pull request #6 : RED-106: replace the local dictionary preload with remove dictionary service. Merge in RED/redaction-service from feature/RED-106i to master * commit 'd607ac567d1e07cfc4c7dd4fa581d1881a874537': RED-106: integrationTest fixed. RED-106: integrationTest fixed. RED-106: integrationTest fixed. REd-106: rebase auf master REd-106: rebase auf master DEV: bugfix missing bean REd-106: enable dictionary version RED-106: replace the local dictionary preload with remove dictionary service. RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-21 13:19:27 +02:00
cheng	d607ac567d	RED-106: integrationTest fixed.	2020-07-21 13:15:58 +02:00
cheng	1b40931c87	RED-106: integrationTest fixed.	2020-07-21 13:14:17 +02:00
cheng	75c6433eef	RED-106: integrationTest fixed.	2020-07-21 13:14:10 +02:00
cheng	9c14f331fd	REd-106: rebase auf master	2020-07-20 10:13:35 +02:00
cheng	141e4b9371	REd-106: rebase auf master	2020-07-20 09:19:54 +02:00
cheng	902d5cddf7	DEV: bugfix missing bean	2020-07-20 09:18:42 +02:00
cheng	b3841b86b1	REd-106: enable dictionary version	2020-07-20 09:18:42 +02:00
cheng	4e3a6af3f8	RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-20 09:18:42 +02:00
cheng	8ed548f1a8	RED-106: replace the local dictionary preload with remove dictionary service.	2020-07-20 09:18:42 +02:00
Thierry Goeckel	fe5c20e1a0	Pull request #7 : Fix application startup and remove redundant code. Merge in RED/redaction-service from bugfix/application-startup to master * commit '3e18fe19f31081df98670130512ab3933f7015af': Fix application startup and remove redundant code.	2020-07-20 09:18:21 +02:00