RED-743: Updated to latest rules and dictionaries

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -66,14 +66,16 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
 public class RedactionIntegrationTest {
 
     private static final String RULES = loadFromClassPath("drools/rules.drl");
-    private static final String VERTEBRATES_CODE = "vertebrate";
+    private static final String VERTEBRATE = "vertebrate";
-    private static final String ADDRESS_CODE = "address";
+    private static final String ADDRESS = "address";
-    private static final String NAME_CODE = "name";
+    private static final String AUTHOR = "author";
     private static final String SPONSOR = "sponsor";
     private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
     private static final String REDACTION_INDICATOR = "redaction_indicator";
     private static final String HINT_ONLY = "hint_only";
     private static final String MUST_REDACT = "must_redact";
+    private static final String PUBLISHED_INFORMATION = "published_information";
+    private static final String TEST_METHOD = "test_method";
 
     @Autowired
     private RedactionController redactionController;
@@ -122,37 +124,39 @@ public class RedactionIntegrationTest {
         loadTypeForTest();
         when(dictionaryClient.getVersion()).thenReturn(0L);
         when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
-        when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
+        when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE));
-        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
+        when(dictionaryClient.getDictionaryForType(ADDRESS)).thenReturn(getDictionaryResponse(ADDRESS));
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
+        when(dictionaryClient.getDictionaryForType(AUTHOR)).thenReturn(getDictionaryResponse(AUTHOR));
         when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR));
         when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
         when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
         when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
         when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
+        when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
+        when(dictionaryClient.getDictionaryForType(TEST_METHOD)).thenReturn(getDictionaryResponse(TEST_METHOD));
         when(dictionaryClient.getColors()).thenReturn(colors);
     }
 
 
     private void loadDictionaryForTest() {
 
-        dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>())
+        dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/names.txt")
+                .addAll(ResourceLoader.load("dictionaries/author.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
         dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/sponsor_companies.txt")
+                .addAll(ResourceLoader.load("dictionaries/sponsor.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
+        dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
+                .addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
-        dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>())
+        dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
-                .addAll(ResourceLoader.load("dictionaries/addresses.txt")
+                .addAll(ResourceLoader.load("dictionaries/address.txt")
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
@@ -176,6 +180,16 @@ public class RedactionIntegrationTest {
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/published_information.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/test_method.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
     }
 
 
@@ -187,32 +201,39 @@ public class RedactionIntegrationTest {
 
     private void loadTypeForTest() {
 
-        typeColorMap.put(VERTEBRATES_CODE, "#00ff00");
+        typeColorMap.put(VERTEBRATE, "#ff85f7");
-        typeColorMap.put(ADDRESS_CODE, "#00ffff");
+        typeColorMap.put(ADDRESS, "#ffe187");
-        typeColorMap.put(NAME_CODE, "#ffff00");
+        typeColorMap.put(AUTHOR, "#ffe187");
-        typeColorMap.put(SPONSOR, "#2c21fc");
+        typeColorMap.put(SPONSOR, "#85ebff");
-        typeColorMap.put(NO_REDACTION_INDICATOR, "#e600ff");
+        typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
-        typeColorMap.put(REDACTION_INDICATOR, "#ff7700");
+        typeColorMap.put(REDACTION_INDICATOR, "#caff85");
-        typeColorMap.put(HINT_ONLY, "#00fcb1");
+        typeColorMap.put(HINT_ONLY, "#abc0c4");
-        typeColorMap.put(MUST_REDACT, "#ff0000");
+        typeColorMap.put(MUST_REDACT, "#fab4c0");
+        typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
+        typeColorMap.put(TEST_METHOD, "#91fae8");
 
-        hintTypeMap.put(VERTEBRATES_CODE, true);
+
-        hintTypeMap.put(ADDRESS_CODE, false);
+        hintTypeMap.put(VERTEBRATE, true);
-        hintTypeMap.put(NAME_CODE, false);
+        hintTypeMap.put(ADDRESS, false);
+        hintTypeMap.put(AUTHOR, false);
         hintTypeMap.put(SPONSOR, false);
         hintTypeMap.put(NO_REDACTION_INDICATOR, true);
         hintTypeMap.put(REDACTION_INDICATOR, true);
         hintTypeMap.put(HINT_ONLY, true);
         hintTypeMap.put(MUST_REDACT, true);
+        hintTypeMap.put(PUBLISHED_INFORMATION, true);
+        hintTypeMap.put(TEST_METHOD, true);
 
-        caseInSensitiveMap.put(VERTEBRATES_CODE, true);
+        caseInSensitiveMap.put(VERTEBRATE, true);
-        caseInSensitiveMap.put(ADDRESS_CODE, false);
+        caseInSensitiveMap.put(ADDRESS, false);
-        caseInSensitiveMap.put(NAME_CODE, false);
+        caseInSensitiveMap.put(AUTHOR, false);
         caseInSensitiveMap.put(SPONSOR, false);
         caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
         caseInSensitiveMap.put(REDACTION_INDICATOR, true);
         caseInSensitiveMap.put(HINT_ONLY, true);
         caseInSensitiveMap.put(MUST_REDACT, true);
+        caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
+        caseInSensitiveMap.put(TEST_METHOD, false);
 
         colors.setDefaultColor("#acfc00");
         colors.setNotRedacted("#cccccc");

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java

@@ -53,7 +53,7 @@ import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationSer
 public class EntityRedactionServiceTest {
 
     private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
-    private static final String NAME_CODE = "name";
+    private static final String AUTHOR_CODE = "author";
     private static final String ADDRESS_CODE = "address";
     private static final String SPONSOR_CODE = "sponsor";
 
@@ -126,12 +126,17 @@ public class EntityRedactionServiceTest {
                 .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
                 .build();
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
                 .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
 
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
+
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -154,11 +159,15 @@ public class EntityRedactionServiceTest {
                 .entries(Arrays.asList("Casey, H.W.", "O’Loughlin,  C.K.", "Salamon, C.M.", "Smith, S.H."))
                 .build();
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
                 .entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
 
         try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@@ -176,13 +185,17 @@ public class EntityRedactionServiceTest {
                 " Supplement - Identity of the active substance - Reference list.pdf");
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
                 .build();
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -209,13 +222,17 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
                 .build();
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor.txt")))
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -274,13 +291,17 @@ public class EntityRedactionServiceTest {
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
                 .build();
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -310,8 +331,16 @@ public class EntityRedactionServiceTest {
 
         ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
+        DictionaryResponse addressResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
+        DictionaryResponse authorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(authorResponse);
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
-                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor_companies.txt")))
+                .entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor.txt")))
                 .build();
         when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(dictionaryResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@@ -336,12 +365,16 @@ public class EntityRedactionServiceTest {
                 .build();
 
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
                 .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
 
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -357,7 +390,7 @@ public class EntityRedactionServiceTest {
                 .build();
 
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         addressResponse = DictionaryResponse.builder()
                 .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
                 .build();
@@ -383,12 +416,16 @@ public class EntityRedactionServiceTest {
                 .build();
 
         when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
                 .entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
 
+        DictionaryResponse sponsorResponse = DictionaryResponse.builder()
+                .entries(Collections.emptyList())
+                .build();
+        when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
         try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
             Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
             entityRedactionService.processDocument(classifiedDoc, null);
@@ -427,7 +464,7 @@ public class EntityRedactionServiceTest {
         when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
         TypeResponse typeResponse = TypeResponse.builder()
                 .types(Arrays.asList(
-                        TypeResult.builder().type(NAME_CODE).hexColor("#ffff00").build(),
+                        TypeResult.builder().type(AUTHOR_CODE).hexColor("#ffff00").build(),
                         TypeResult.builder().type(ADDRESS_CODE).hexColor("#ff00ff").build(),
                         TypeResult.builder().type(SPONSOR_CODE).hexColor("#00ffff").build()))
                 .build();
@@ -437,7 +474,7 @@ public class EntityRedactionServiceTest {
         // Default empty return to prevent NPEs
         DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
                 .build();
-        when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
+        when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
         DictionaryResponse addressResponse = DictionaryResponse.builder()
                 .build();
         when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt

@@ -1 +0,0 @@
-unpublished

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt

@@ -1,2 +1 @@
-CTL
 determination of residues

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt

@@ -1,3 +1,4 @@
-published paper
+Fundamentals of Applied Toxicology
 in vitro
 in-vitro
+published paper

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt

@@ -0,0 +1,87 @@
+7th ed.; The Iowa State University Press: Ames, IA
+Analytical chemistry
+Animal Reproduction
+Animal Reproduction Science
+Apidologie
+Aquatic toxicology
+Archives of Environmental Contamination and Toxicology
+ATLA
+Atmospheric. Environment
+Australasian Journal of Ecotoxicology
+Background lesions of laboratory Animals, A Color Atlas
+Biometrics
+Biometrika
+Birth Defects Res. B. Dev. Reprod. Toxicol.
+Crit Rev Toxicol
+Current approaches in the statistical analysis of ecotoxicity data: guidance to application
+Curr. Med. Chem.
+Dongbei Nongye Daxue Xuebao
+Ecotoxicology and Environmental Safety
+Environ and Molecular Mutagenesis
+Environ. Health Perspec.
+Environ Health Perspect
+Environ Health Perspect.
+Environ Health Perspect. 1
+Environmental Health
+Environmental Health Perspectives
+Environmental & Molecular Mutagenesis
+Environmental Pollution
+Environmental Science
+Environmental Science and Technology.
+Environmental Science & Technology
+Environmental Toxicology and Chemistry
+Environ Mol Muta- gen
+Environ. Sci. Technol.
+Environ Toxicol.
+Env. Mol. Mutagen
+Erna¨hrung
+Essays in Honor of Harold Hotelling
+Fish Sci
+Food Cosmet. Toxicol.
+Fundamentals of Applied Toxicology
+Fundamentals of Applied Toxicology1988
+High-Throughput Screening Methods in Toxicity Testing
+High-Throughput Screening Methods in Toxicity Testing. Hoboken, NJ: John Wiley & Sons
+http://www.iobc-wprs.org
+Irish Journal of Agricultural and Food Research
+J Econom
+J Endocrinol
+J. Invest. Derm
+Journal of agricultural and food chemistry
+Journal of Applied Entomology
+Journal of Environmental Science and Health
+Journal of Experimental Biology and Ecology
+Journal of Hazardous Materials
+Journal of the American College of Toxicology
+Journal of Toxicology and
+J Steroid Biochem Mol Biol
+Marine Pollution Bulletin
+Middle Atlantic Reproduction and Teratology Association
+Mol. Cell. Endocrinol.
+Mol Mutagen
+Mutagenesis
+Mutat Res
+Nonparametric Statistics for the Behavioral Sciences
+Principles and Procedures of Statistics
+Principles and Procedures of Statistics, A Biometrical Approach
+Proc Natl Acad Sci USA
+Progress in
+Psychopharmacologia
+Reproductive Toxicology
+RNA
+Science of the Total Environment
+Stain Technol
+Statistical Methods
+Statistical Methods,
+Steinberg P,
+Teratology
+The American Statistician
+Toxicol Chem
+Toxicol in Vitro
+Toxicological and Environmental Chemistry
+Toxicological Sciences
+Toxicologic Pathology
+Toxicology and Applied Pharmacology
+Toxicol Sci
+Toxicol Sci.
+Toxicol Sci. 1

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt

@@ -1,9 +1,12 @@
-in vivo
-in-vivo
-dermal penetration
-oral toxicity
-oral-toxicity
 acute toxicity
 acute-toxicity
+dermal penetration
 eco toxicity
 eco-toxicity
+Env. Mol. Mutagen,
+in vivo
+in-vivo
+ld
+oral toxicity
+oral-toxicity
+Processes & Impacts

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor.txt

@@ -1,7 +1,8 @@
+Crumrine
+Fine Organics Limited, Middlesbrough, United Kingdom
+Hunan Haili Chemical Industry Co., Ltd., Hunan, China
 Monthey Syngenta Crop Protection AG, Basel, Switzerland
 Syngenta Crop Protection, Monthey, Switzerland
-Fine Organics Limited, Middlesbrough, United Kingdom
 Syngenta Monthey Switzerland
-Hunan Haili Chemical Industry Co., Ltd., Hunan, China
-Syngenta, Switzerland
 Syngenta Nantong, China
+Syngenta, Switzerland

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/test_method.txt

@@ -0,0 +1,48 @@
+Bartlett’s Test
+Bonferroni-Holm Adjustment
+Cochran-Armitage test
+Cochran-Armitage Trend Step-Down Test
+Dunnett's Multiple Comparison test
+Dunnett’s test
+Dunnett's "t" test
+Dunnett test
+Fisher Count-AllTM c
+Fisher Scientific Company
+Fisher’s Exact
+Fisher's exact test
+Fisher’s Exact Test
+Ham's F10
+Heinz body
+Heinz body determinations
+Jonckheere-Terpstra test (step-down)
+Jonckheere Trend Test
+Kay and Calandra system
+Klimisch code
+Kolmogorov-Smirnov's test
+Krebs cycle
+Litchfield and Wilcoxon method
+Magnusson and Kligman
+Magnusson & Kligman
+MAGNUSSON & KLINGMAN METHOD
+Mann-Whitney's test
+Maximisation test, Buehler
+Method of Magnusson and Kligman
+methodology validated at Smithers Viscient.
+n Williams' medium E a
+Patterson-Kelly twinshell blender
+Shapiro-Wilks and Levene’s tests
+Shapiro-Wilks’ Test
+Steel’s test
+Tamhane-Dunnett
+Tamhane-Dunnett test
+the method Davies
+Vogel-Bonner Medium E
+Welch’s test
+Welch t-test
+Whitney test
+Wilcoxon and Ansari-Bradley statistics
+Williams multiple sequential t-test
+Williams' test
+Williams t-test
+William test
+Winkler titration technique,

redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrate.txt

@@ -1,4 +1,3 @@
-a. sylvaticus
 african clawed frog
 agalychnis callidryas
 albino rat
@@ -15,6 +14,7 @@ apodemus flavicollis
 apodemus syl vaticus
 apodemus sylvaticus
 arvicola terrestris
+a. sylvaticus
 avian
 bank vole
 bird
@@ -23,6 +23,7 @@ bluegill
 bluegill sunfish
 bobwhite
 bobwhite quail
+Bovine
 brachydanio rerio
 brown hare
 bufo americanus
@@ -52,10 +53,12 @@ crocidura russula
 crucian carp
 cyprinodon variegatus
 cyprinus carpio
+(Danio rerio
 dog
 dogs
 duck
 ducks
+Environmental & Molecular Mutagenesis
 european brown hare
 european rabbit
 fathead minnow
@@ -121,14 +124,13 @@ myodes glareolus
 northern bobwhite
 o. cuniculus
 o. mykiss
-o. tshawytscha
 oncorhynchus
 oncorhynchus mykiss
 oncorhynchus tshawytscha
 oryctolagus cuniculus
 oryzias melastigma
 oryzias melastigma larvae
-p. promelas
+o. tshawytscha
 pagrus major
 palumbus
 pig
@@ -139,10 +141,10 @@ pimephales promela
 pimephales promelas
 poecilia reticulata
 poultry
+p. promelas
 pseudacris
 pseudacris triseriata
 quail
-r. catesbeiana
 rabbit
 rabbits
 rainbow trout
@@ -152,6 +154,7 @@ rana limnocharis
 rana pipiens
 rat
 rats
+r. catesbeiana
 reptile
 reptiles
 ricefish
@@ -164,10 +167,12 @@ sheepshead minnow
 sheepshead minnows
 shrew
 shrews
+Singh
 sorex araneus
 spea multiplicata
 spotted march frog
 tadpoles
+Taeniopygia guttata
 terrestrial vertrebrates
 toad
 treefrog
@@ -187,5 +192,6 @@ wood pigeon
 xenopus laevis
 xenpous leavis
 yellow-necked mouse
+Zebra Finch
 zebra fish
 zebrafish

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -9,7 +9,7 @@ rule "1: Redacted because Section contains Vertebrate"
 	when
 		Section(matchesType("vertebrate"))
 	then
-		section.redact("name", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redact("author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 		section.redact("address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 	end
 
@@ -18,7 +18,7 @@ rule "2: Not Redacted because Section contains no Vertebrate"
 	when
 		Section(!matchesType("vertebrate"))
 	then
-		section.redactNot("name", 2, "No Vertebrate found");
+		section.redactNot("author", 2, "No Vertebrate found");
 		section.redactNot("address", 2, "No Vertebrate found");
 	end
 
@@ -27,7 +27,7 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
 	when
 		Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
 	then
-		section.redactNot("name", 3, "Vertebrate and No Redaction Indicator found");
+		section.redactNot("author", 3, "Vertebrate and No Redaction Indicator found");
 		section.redactNot("address", 3, "Vertebrate and No Redaction Indicator found");
 	end
 
@@ -36,7 +36,7 @@ rule "4: Do not redact Names and Addresses if no redaction Indicator is containe
 	when
 		Section(matchesType("vertebrate"), matchesType("published_information"))
 	then
-		section.redactNot("name", 4, "Vertebrate and Published Information found");
+		section.redactNot("author", 4, "Vertebrate and Published Information found");
 		section.redactNot("address", 4, "Vertebrate and Published Information found");
 	end
 
@@ -45,7 +45,7 @@ rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indi
 	when
 		Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
 	then
-		section.redact("name", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redact("author", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 		section.redact("address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 	end
 
@@ -97,7 +97,7 @@ rule "8: Not redacted because Vertebrate Study = N"
     when
         Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
     then
-        section.redactNotCell("Author(s)", 8, "name", "Not redacted because row is not a vertebrate study");
+        section.redactNotCell("Author(s)", 8, "author", "Not redacted because row is not a vertebrate study");
         section.redactNot("address", 8, "Not redacted because row is not a vertebrate study");
         section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
     end
@@ -107,7 +107,7 @@ rule "9: Redact if must redact entry is found"
 	when
 		Section(matchesType("must_redact"))
 	then
-		section.redact("name", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+		section.redact("author", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 		section.redact("address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
 	end
 
@@ -116,7 +116,7 @@ rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate
     when
         Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
     then
-        section.redactCell("Author(s)", 10, "name", "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
+        section.redactCell("Author(s)", 10, "author", "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
         section.redact("address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
         section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
     end