RED-211, RED-215 Added dictionaries and rules for testing.

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -19,7 +19,6 @@ import java.util.stream.Collectors;
 
 import org.apache.commons.io.IOUtils;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.kie.api.KieServices;
@@ -48,7 +47,6 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader;
 import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities;
 
-@Ignore
 @RunWith(SpringRunner.class)
 @SpringBootTest(webEnvironment = DEFINED_PORT)
 public class RedactionIntegrationTest {
@@ -58,6 +56,8 @@ public class RedactionIntegrationTest {
     private static final String ADDRESS_CODE = "address";
     private static final String NAME_CODE = "name";
     private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
+    private static final String REDACTION_INDICATOR = "redaction_indicator";
+    private static final String HINT_ONLY = "hint_only";
 
     @Autowired
     private RedactionController redactionController;
@@ -109,6 +109,8 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
         when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
         when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
+        when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
         when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f}));
     }
 
@@ -131,7 +133,17 @@ public class RedactionIntegrationTest {
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt")
+                .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/hint_only.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
@@ -149,17 +161,23 @@ public class RedactionIntegrationTest {
         typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
         typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
         typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
-        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0});
+        typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f});
+        typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f});
+        typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f});
 
         hintTypeMap.put(VERTEBRATES_CODE, true);
         hintTypeMap.put(ADDRESS_CODE, false);
         hintTypeMap.put(NAME_CODE, false);
         hintTypeMap.put(NO_REDACTION_INDICATOR, true);
+        hintTypeMap.put(REDACTION_INDICATOR, true);
+        hintTypeMap.put(HINT_ONLY, true);
 
         caseInSensitiveMap.put(VERTEBRATES_CODE, true);
         caseInSensitiveMap.put(ADDRESS_CODE, false);
         caseInSensitiveMap.put(NAME_CODE, false);
         caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(REDACTION_INDICATOR, true);
+        caseInSensitiveMap.put(HINT_ONLY, true);
     }
 
 

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt

@@ -1 +0,0 @@
-In Vitro

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt

@@ -0,0 +1,2 @@
+guideline
+unpublished

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt

@@ -674,6 +674,7 @@ Barnes E
 Blunt H
 Bohnenberger S
 Broich K
+Broich
 Cords SM
 Cowie D
 Davies S
@@ -710,6 +711,8 @@ Randall R
 Manton J
 Nagy K
 Petus–Árpásy M
+Petus–Árpásy
+Petus-Árpásy
 Roth M
 Shearer J
 Sieber M
@@ -2380,6 +2383,7 @@ Bowles
 Dollenmeier
 Myhr
 Chang et al
+Chang
 Beck
 Orton
 Medjakovic

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt

@@ -0,0 +1,3 @@
+published paper
+in vitro
+in-vitro

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt

@@ -0,0 +1,9 @@
+in vivo
+in-vivo
+dermal penetration
+oral toxicity
+oral-toxicity
+acute toxicity
+acute-toxicity
+eco toxicity
+eco-toxicity

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt

@@ -227,3 +227,4 @@ Guinea-pigs
 White rabbits
 Birds
 Wood mice
+Mallard

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -32,27 +32,47 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
 	end
 
 
-rule "4: Redact contact information, if applicant is found"
+rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
 	when
-		eval(section.getText().toLowerCase().contains("applicant"));
+		eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true);
 	then
-		section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4");
+		section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
-		section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4");
+		section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
-		section.redactLineAfter("Contact point:",  "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4");
-		section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4");
 	end
 
 
-rule "5: Redact contact information, if 'Producer of the plant protection product' is found"
+rule "5: Do not redact in guideline sections"
+	when
+		eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance"));
+	then
+		section.redactNot("name", 5, "Section is a guideline section.");
+		section.redactNot("address", 5, "Section is a guideline section.");
+	end
+
+
+rule "6: Redact contact information, if applicant is found"
+	when
+		eval(section.getText().toLowerCase().contains("applicant") == true);
+	then
+		section.redactLineAfter("Name:", "address", 6, "contact information was found");
+		section.redactBetween("Address:", "Contact", "address", 6, "contact information was found");
+		section.redactLineAfter("Contact point:",  "address", 6, "contact information was found");
+		section.redactLineAfter("Phone:", "address", 6, "contact information was found");
+		section.redactLineAfter("Fax:", "address", 6, "contact information was found");
+		section.redactLineAfter("E-mail:", "address", 6, "contact information was found");
+		section.redactLineAfter("Contact:", "address", 6, "contact information was found");
+		section.redactLineAfter("Telephone number:", "address", 6, "contact information was found");
+	end
+
+
+rule "7: Redact contact information, if 'Producer of the plant protection product' is found"
 	when
 		eval(section.getText().contains("Producer of the plant protection product"));
 	then
-		section.redactLineAfter("Name:", "address", 5, "xxxx");
+		section.redactLineAfter("Name:", "address", 7, "Producer of the plant protection product was found");
-		section.redactBetween("Address:", "Contact", "address", 5, "xxxx");
+		section.redactBetween("Address:", "Contact", "address", 7, "Producer of the plant protection product was found");
-		section.redactBetween("Contact:", "Phone", "address", 5, "xxxx");
+		section.redactBetween("Contact:", "Phone", "address", 7, "Producer of the plant protection product was found");
-		section.redactLineAfter("Phone:", "address", 5, "xxxx");
+		section.redactLineAfter("Phone:", "address", 7, "Producer of the plant protection product was found");
-		section.redactLineAfter("Fax:", "address", 5, "xxxx");
+		section.redactLineAfter("Fax:", "address", 7, "Producer of the plant protection product was found");
-		section.redactLineAfter("E-mail:", "address", 5, "xxxx");
+		section.redactLineAfter("E-mail:", "address", 7, "Producer of the plant protection product was found");
 	end