From e2895a1c7ace69f0fa3363af7d931480d63dca24 Mon Sep 17 00:00:00 2001 From: deiflaender Date: Fri, 31 Jul 2020 16:21:49 +0200 Subject: [PATCH] RED-211, RED-215 Added dictionaries and rules for testing. --- .../v1/server/RedactionIntegrationTest.java | 26 ++++++++-- .../dictionaries/NoRedactionIndicator.txt | 1 - .../test/resources/dictionaries/hint_only.txt | 2 + .../src/test/resources/dictionaries/names.txt | 4 ++ .../dictionaries/no_redaction_indicator.txt | 3 ++ .../dictionaries/redaction_indicator.txt | 9 ++++ .../resources/dictionaries/vertebrates.txt | 3 +- .../src/test/resources/drools/rules.drl | 50 +++++++++++++------ 8 files changed, 77 insertions(+), 21 deletions(-) delete mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt create mode 100644 redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 400bf046..015b3a25 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -19,7 +19,6 @@ import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.kie.api.KieServices; @@ -48,7 +47,6 @@ import com.iqser.red.service.redaction.v1.server.controller.RedactionController; import com.iqser.red.service.redaction.v1.server.redaction.utils.ResourceLoader; import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizationUtilities; -@Ignore @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = DEFINED_PORT) public class RedactionIntegrationTest { @@ -58,6 +56,8 @@ public class RedactionIntegrationTest { private static final String ADDRESS_CODE = "address"; private static final String NAME_CODE = "name"; private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; + private static final String REDACTION_INDICATOR = "redaction_indicator"; + private static final String HINT_ONLY = "hint_only"; @Autowired private RedactionController redactionController; @@ -109,6 +109,8 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE)); when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR)); + when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR)); + when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY)); when(dictionaryClient.getDefaultColor()).thenReturn(new DefaultColor(new float[]{1f, 0.502f, 0f})); } @@ -131,7 +133,17 @@ public class RedactionIntegrationTest { .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); dictionary.computeIfAbsent(NO_REDACTION_INDICATOR, v -> new ArrayList<>()) - .addAll(ResourceLoader.load("dictionaries/NoRedactionIndicator.txt") + .addAll(ResourceLoader.load("dictionaries/no_redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); + dictionary.computeIfAbsent(REDACTION_INDICATOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/redaction_indicator.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); + dictionary.computeIfAbsent(HINT_ONLY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/hint_only.txt") .stream() .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); @@ -149,17 +161,23 @@ public class RedactionIntegrationTest { typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0}); typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1}); typeColorMap.put(NAME_CODE, new float[]{1, 1, 0}); - typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{1, 0.502f, 0}); + typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f}); + typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f}); + typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f}); hintTypeMap.put(VERTEBRATES_CODE, true); hintTypeMap.put(ADDRESS_CODE, false); hintTypeMap.put(NAME_CODE, false); hintTypeMap.put(NO_REDACTION_INDICATOR, true); + hintTypeMap.put(REDACTION_INDICATOR, true); + hintTypeMap.put(HINT_ONLY, true); caseInSensitiveMap.put(VERTEBRATES_CODE, true); caseInSensitiveMap.put(ADDRESS_CODE, false); caseInSensitiveMap.put(NAME_CODE, false); caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); + caseInSensitiveMap.put(REDACTION_INDICATOR, true); + caseInSensitiveMap.put(HINT_ONLY, true); } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt deleted file mode 100644 index def512e4..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/NoRedactionIndicator.txt +++ /dev/null @@ -1 +0,0 @@ -In Vitro \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt new file mode 100644 index 00000000..bcb25021 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt @@ -0,0 +1,2 @@ +guideline +unpublished \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt index 71a22a19..148b36c9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt @@ -674,6 +674,7 @@ Barnes E Blunt H Bohnenberger S Broich K +Broich Cords SM Cowie D Davies S @@ -710,6 +711,8 @@ Randall R Manton J Nagy K Petus–Árpásy M +Petus–Árpásy +Petus-Árpásy Roth M Shearer J Sieber M @@ -2380,6 +2383,7 @@ Bowles Dollenmeier Myhr Chang et al +Chang Beck Orton Medjakovic diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt new file mode 100644 index 00000000..92b673c6 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt @@ -0,0 +1,3 @@ +published paper +in vitro +in-vitro \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt new file mode 100644 index 00000000..b498e73c --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt @@ -0,0 +1,9 @@ +in vivo +in-vivo +dermal penetration +oral toxicity +oral-toxicity +acute toxicity +acute-toxicity +eco toxicity +eco-toxicity \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt index 6337a8b0..82c451e0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt @@ -226,4 +226,5 @@ Poultry Guinea-pigs White rabbits Birds -Wood mice \ No newline at end of file +Wood mice +Mallard \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 74f52589..44421df1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -32,27 +32,47 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe end -rule "4: Redact contact information, if applicant is found" +rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" when - eval(section.getText().toLowerCase().contains("applicant")); + eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true); then - section.redactLineAfter("Name:", "address", 4, "Redacted because of Rule 4"); - section.redactBetween("Address:", "Contact", "address", 4, "Redacted because of Rule 4"); - section.redactLineAfter("Contact point:", "address", 4, "Redacted because of Rule 4"); - section.redactLineAfter("Phone:", "address", 4, "Redacted because of Rule 4"); - section.redactLineAfter("Fax:", "address", 4, "Redacted because of Rule 4"); - section.redactLineAfter("E-mail:", "address", 4, "Redacted because of Rule 4"); + section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator"); + section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator"); end -rule "5: Redact contact information, if 'Producer of the plant protection product' is found" +rule "5: Do not redact in guideline sections" + when + eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance")); + then + section.redactNot("name", 5, "Section is a guideline section."); + section.redactNot("address", 5, "Section is a guideline section."); + end + + +rule "6: Redact contact information, if applicant is found" + when + eval(section.getText().toLowerCase().contains("applicant") == true); + then + section.redactLineAfter("Name:", "address", 6, "contact information was found"); + section.redactBetween("Address:", "Contact", "address", 6, "contact information was found"); + section.redactLineAfter("Contact point:", "address", 6, "contact information was found"); + section.redactLineAfter("Phone:", "address", 6, "contact information was found"); + section.redactLineAfter("Fax:", "address", 6, "contact information was found"); + section.redactLineAfter("E-mail:", "address", 6, "contact information was found"); + section.redactLineAfter("Contact:", "address", 6, "contact information was found"); + section.redactLineAfter("Telephone number:", "address", 6, "contact information was found"); + end + + +rule "7: Redact contact information, if 'Producer of the plant protection product' is found" when eval(section.getText().contains("Producer of the plant protection product")); then - section.redactLineAfter("Name:", "address", 5, "xxxx"); - section.redactBetween("Address:", "Contact", "address", 5, "xxxx"); - section.redactBetween("Contact:", "Phone", "address", 5, "xxxx"); - section.redactLineAfter("Phone:", "address", 5, "xxxx"); - section.redactLineAfter("Fax:", "address", 5, "xxxx"); - section.redactLineAfter("E-mail:", "address", 5, "xxxx"); + section.redactLineAfter("Name:", "address", 7, "Producer of the plant protection product was found"); + section.redactBetween("Address:", "Contact", "address", 7, "Producer of the plant protection product was found"); + section.redactBetween("Contact:", "Phone", "address", 7, "Producer of the plant protection product was found"); + section.redactLineAfter("Phone:", "address", 7, "Producer of the plant protection product was found"); + section.redactLineAfter("Fax:", "address", 7, "Producer of the plant protection product was found"); + section.redactLineAfter("E-mail:", "address", 7, "Producer of the plant protection product was found"); end \ No newline at end of file