Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-743: Updated to latest rules and dictionaries

Browse Source
This commit is contained in:
deiflaender 2020-11-23 11:10:16 +01:00
parent 5481dfbcff
commit 6de87d051e
13 changed files with 2507 additions and 1970 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -66,14 +66,16 @@ import com.iqser.red.service.redaction.v1.server.redaction.utils.TextNormalizati
public class RedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/rules.drl");
private static final String VERTEBRATES_CODE = "vertebrate";
private static final String ADDRESS_CODE = "address";
private static final String NAME_CODE = "name";
private static final String VERTEBRATE = "vertebrate";
private static final String ADDRESS = "address";
private static final String AUTHOR = "author";
private static final String SPONSOR = "sponsor";
private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
private static final String REDACTION_INDICATOR = "redaction_indicator";
private static final String HINT_ONLY = "hint_only";
private static final String MUST_REDACT = "must_redact";
private static final String PUBLISHED_INFORMATION = "published_information";
private static final String TEST_METHOD = "test_method";
@Autowired
private RedactionController redactionController;
@ -122,37 +124,39 @@ public class RedactionIntegrationTest {
loadTypeForTest();
when(dictionaryClient.getVersion()).thenReturn(0L);
when(dictionaryClient.getAllTypes()).thenReturn(TypeResponse.builder().types(getTypeResponse()).build());
when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
when(dictionaryClient.getDictionaryForType(VERTEBRATE)).thenReturn(getDictionaryResponse(VERTEBRATE));
when(dictionaryClient.getDictionaryForType(ADDRESS)).thenReturn(getDictionaryResponse(ADDRESS));
when(dictionaryClient.getDictionaryForType(AUTHOR)).thenReturn(getDictionaryResponse(AUTHOR));
when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
when(dictionaryClient.getDictionaryForType(MUST_REDACT)).thenReturn(getDictionaryResponse(MUST_REDACT));
when(dictionaryClient.getDictionaryForType(PUBLISHED_INFORMATION)).thenReturn(getDictionaryResponse(PUBLISHED_INFORMATION));
when(dictionaryClient.getDictionaryForType(TEST_METHOD)).thenReturn(getDictionaryResponse(TEST_METHOD));
when(dictionaryClient.getColors()).thenReturn(colors);
}
private void loadDictionaryForTest() {
dictionary.computeIfAbsent(NAME_CODE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/names.txt")
dictionary.computeIfAbsent(AUTHOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/author.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/sponsor_companies.txt")
.addAll(ResourceLoader.load("dictionaries/sponsor.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
dictionary.computeIfAbsent(VERTEBRATE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrate.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(ADDRESS_CODE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/addresses.txt")
dictionary.computeIfAbsent(ADDRESS, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/address.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
@ -176,6 +180,16 @@ public class RedactionIntegrationTest {
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PUBLISHED_INFORMATION, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/published_information.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(TEST_METHOD, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/test_method.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
}
@ -187,32 +201,39 @@ public class RedactionIntegrationTest {
private void loadTypeForTest() {
typeColorMap.put(VERTEBRATES_CODE, "#00ff00");
typeColorMap.put(ADDRESS_CODE, "#00ffff");
typeColorMap.put(NAME_CODE, "#ffff00");
typeColorMap.put(SPONSOR, "#2c21fc");
typeColorMap.put(NO_REDACTION_INDICATOR, "#e600ff");
typeColorMap.put(REDACTION_INDICATOR, "#ff7700");
typeColorMap.put(HINT_ONLY, "#00fcb1");
typeColorMap.put(MUST_REDACT, "#ff0000");
typeColorMap.put(VERTEBRATE, "#ff85f7");
typeColorMap.put(ADDRESS, "#ffe187");
typeColorMap.put(AUTHOR, "#ffe187");
typeColorMap.put(SPONSOR, "#85ebff");
typeColorMap.put(NO_REDACTION_INDICATOR, "#be85ff");
typeColorMap.put(REDACTION_INDICATOR, "#caff85");
typeColorMap.put(HINT_ONLY, "#abc0c4");
typeColorMap.put(MUST_REDACT, "#fab4c0");
typeColorMap.put(PUBLISHED_INFORMATION, "#85ebff");
typeColorMap.put(TEST_METHOD, "#91fae8");
hintTypeMap.put(VERTEBRATES_CODE, true);
hintTypeMap.put(ADDRESS_CODE, false);
hintTypeMap.put(NAME_CODE, false);
hintTypeMap.put(VERTEBRATE, true);
hintTypeMap.put(ADDRESS, false);
hintTypeMap.put(AUTHOR, false);
hintTypeMap.put(SPONSOR, false);
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
hintTypeMap.put(REDACTION_INDICATOR, true);
hintTypeMap.put(HINT_ONLY, true);
hintTypeMap.put(MUST_REDACT, true);
hintTypeMap.put(PUBLISHED_INFORMATION, true);
hintTypeMap.put(TEST_METHOD, true);
caseInSensitiveMap.put(VERTEBRATES_CODE, true);
caseInSensitiveMap.put(ADDRESS_CODE, false);
caseInSensitiveMap.put(NAME_CODE, false);
caseInSensitiveMap.put(VERTEBRATE, true);
caseInSensitiveMap.put(ADDRESS, false);
caseInSensitiveMap.put(AUTHOR, false);
caseInSensitiveMap.put(SPONSOR, false);
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
caseInSensitiveMap.put(HINT_ONLY, true);
caseInSensitiveMap.put(MUST_REDACT, true);
caseInSensitiveMap.put(PUBLISHED_INFORMATION, true);
caseInSensitiveMap.put(TEST_METHOD, false);
colors.setDefaultColor("#acfc00");
colors.setNotRedacted("#cccccc");

73
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java
View File

@ -53,7 +53,7 @@ import com.iqser.red.service.redaction.v1.server.segmentation.PdfSegmentationSer
public class EntityRedactionServiceTest {
private static final String DEFAULT_RULES = loadFromClassPath("drools/rules.drl");
private static final String NAME_CODE = "name";
private static final String AUTHOR_CODE = "author";
private static final String ADDRESS_CODE = "address";
private static final String SPONSOR_CODE = "sponsor";
@ -126,12 +126,17 @@ public class EntityRedactionServiceTest {
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
.build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -154,11 +159,15 @@ public class EntityRedactionServiceTest {
.entries(Arrays.asList("Casey, H.W.", "O’Loughlin, C.K.", "Salamon, C.M.", "Smith, S.H."))
.build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Toxigenics, Inc., Decatur, IL 62526, USA"))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(new ByteArrayInputStream(redactionRequest.getDocument()))) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
@ -176,13 +185,17 @@ public class EntityRedactionServiceTest {
" Supplement - Identity of the active substance - Reference list.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
.build();
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -209,13 +222,17 @@ public class EntityRedactionServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Row With Ambiguous Redaction.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
.build();
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor.txt")))
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -274,13 +291,17 @@ public class EntityRedactionServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/names.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/author.txt")))
.build();
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/addresses.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/address.txt")))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -310,8 +331,16 @@ public class EntityRedactionServiceTest {
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/batches_new_line.pdf");
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse authorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(authorResponse);
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor_companies.txt")))
.entries(new ArrayList<>(ResourceLoader.load("dictionaries/sponsor.txt")))
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(dictionaryResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
@ -336,12 +365,16 @@ public class EntityRedactionServiceTest {
.build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -357,7 +390,7 @@ public class EntityRedactionServiceTest {
.build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build();
@ -383,12 +416,16 @@ public class EntityRedactionServiceTest {
.build();
when(dictionaryClient.getVersion()).thenReturn(DICTIONARY_VERSION.incrementAndGet());
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.entries(Collections.singletonList("Novartis Crop Protection AG, Basel, Switzerland"))
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);
DictionaryResponse sponsorResponse = DictionaryResponse.builder()
.entries(Collections.emptyList())
.build();
when(dictionaryClient.getDictionaryForType(SPONSOR_CODE)).thenReturn(sponsorResponse);
try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) {
Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument);
entityRedactionService.processDocument(classifiedDoc, null);
@ -427,7 +464,7 @@ public class EntityRedactionServiceTest {
when(rulesClient.getRules()).thenReturn(new RulesResponse(tableRules));
TypeResponse typeResponse = TypeResponse.builder()
.types(Arrays.asList(
TypeResult.builder().type(NAME_CODE).hexColor("#ffff00").build(),
TypeResult.builder().type(AUTHOR_CODE).hexColor("#ffff00").build(),
TypeResult.builder().type(ADDRESS_CODE).hexColor("#ff00ff").build(),
TypeResult.builder().type(SPONSOR_CODE).hexColor("#00ffff").build()))
.build();
@ -437,7 +474,7 @@ public class EntityRedactionServiceTest {
// Default empty return to prevent NPEs
DictionaryResponse dictionaryResponse = DictionaryResponse.builder()
.build();
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(dictionaryResponse);
when(dictionaryClient.getDictionaryForType(AUTHOR_CODE)).thenReturn(dictionaryResponse);
DictionaryResponse addressResponse = DictionaryResponse.builder()
.build();
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(addressResponse);

1079
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/addresses.txt → redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/address.txt
View File

File diff suppressed because it is too large Load Diff

3052
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/names.txt → redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/author.txt
View File

File diff suppressed because it is too large Load Diff

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/hint_only.txt
View File

@ -1 +0,0 @@
unpublished

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt
View File

@ -1,2 +1 @@
CTL
determination of residues
determination of residues

5
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/no_redaction_indicator.txt
View File

@ -1,3 +1,4 @@
published paper
Fundamentals of Applied Toxicology
in vitro
in-vitro
in-vitro
published paper

87
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/published_information.txt Normal file
View File

@ -0,0 +1,87 @@
7th ed.; The Iowa State University Press: Ames, IA
Analytical chemistry
Animal Reproduction
Animal Reproduction Science
Apidologie
Aquatic toxicology
Archives of Environmental Contamination and Toxicology
ATLA
Atmospheric. Environment
Australasian Journal of Ecotoxicology
Background lesions of laboratory Animals, A Color Atlas
Biometrics
Biometrika
Birth Defects Res. B. Dev. Reprod. Toxicol.
Crit Rev Toxicol
Current approaches in the statistical analysis of ecotoxicity data: guidance to application
Curr. Med. Chem.
Dongbei Nongye Daxue Xuebao
Ecotoxicology and Environmental Safety
Environ and Molecular Mutagenesis
Environ. Health Perspec.
Environ Health Perspect
Environ Health Perspect.
Environ Health Perspect. 1
Environmental Health
Environmental Health Perspectives
Environmental & Molecular Mutagenesis
Environmental Pollution
Environmental Science
Environmental Science and Technology.
Environmental Science & Technology
Environmental Toxicology and Chemistry
Environ Mol Muta- gen
Environ. Sci. Technol.
Environ Toxicol.
Env. Mol. Mutagen
Erna¨hrung
Essays in Honor of Harold Hotelling
Fish Sci
Food Cosmet. Toxicol.
Fundamentals of Applied Toxicology
Fundamentals of Applied Toxicology1988
High-Throughput Screening Methods in Toxicity Testing
High-Throughput Screening Methods in Toxicity Testing. Hoboken, NJ: John Wiley & Sons
http://www.iobc-wprs.org
Irish Journal of Agricultural and Food Research
J Econom
J Endocrinol
J. Invest. Derm
Journal of agricultural and food chemistry
Journal of Applied Entomology
Journal of Environmental Science and Health
Journal of Experimental Biology and Ecology
Journal of Hazardous Materials
Journal of the American College of Toxicology
Journal of Toxicology and
J Steroid Biochem Mol Biol
Marine Pollution Bulletin
Middle Atlantic Reproduction and Teratology Association
Mol. Cell. Endocrinol.
Mol Mutagen
Mutagenesis
Mutat Res
Nonparametric Statistics for the Behavioral Sciences
Principles and Procedures of Statistics
Principles and Procedures of Statistics, A Biometrical Approach
Proc Natl Acad Sci USA
Progress in
Psychopharmacologia
Reproductive Toxicology
RNA
Science of the Total Environment
Stain Technol
Statistical Methods
Statistical Methods,
Steinberg P,
Teratology
The American Statistician
Toxicol Chem
Toxicol in Vitro
Toxicological and Environmental Chemistry
Toxicological Sciences
Toxicologic Pathology
Toxicology and Applied Pharmacology
Toxicol Sci
Toxicol Sci.
Toxicol Sci. 1

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/redaction_indicator.txt
View File

@ -1,9 +1,12 @@
in vivo
in-vivo
dermal penetration
oral toxicity
oral-toxicity
acute toxicity
acute-toxicity
dermal penetration
eco toxicity
eco-toxicity
eco-toxicity
Env. Mol. Mutagen,
in vivo
in-vivo
ld
oral toxicity
oral-toxicity
Processes & Impacts

7
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor_companies.txt → redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor.txt
View File

@ -1,7 +1,8 @@
Crumrine
Fine Organics Limited, Middlesbrough, United Kingdom
Hunan Haili Chemical Industry Co., Ltd., Hunan, China
Monthey Syngenta Crop Protection AG, Basel, Switzerland
Syngenta Crop Protection, Monthey, Switzerland
Fine Organics Limited, Middlesbrough, United Kingdom
Syngenta Monthey Switzerland
Hunan Haili Chemical Industry Co., Ltd., Hunan, China
Syngenta Nantong, China
Syngenta, Switzerland
Syngenta Nantong, China

48
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/test_method.txt Normal file
View File

@ -0,0 +1,48 @@
Bartlett’s Test
Bonferroni-Holm Adjustment
Cochran-Armitage test
Cochran-Armitage Trend Step-Down Test
Dunnett's Multiple Comparison test
Dunnett’s test
Dunnett's "t" test
Dunnett test
Fisher Count-AllTM c
Fisher Scientific Company
Fisher’s Exact
Fisher's exact test
Fisher’s Exact Test
Ham's F10
Heinz body
Heinz body determinations
Jonckheere-Terpstra test (step-down)
Jonckheere Trend Test
Kay and Calandra system
Klimisch code
Kolmogorov-Smirnov's test
Krebs cycle
Litchfield and Wilcoxon method
Magnusson and Kligman
Magnusson & Kligman
MAGNUSSON & KLINGMAN METHOD
Mann-Whitney's test
Maximisation test, Buehler
Method of Magnusson and Kligman
methodology validated at Smithers Viscient.
n Williams' medium E a
Patterson-Kelly twinshell blender
Shapiro-Wilks and Levene’s tests
Shapiro-Wilks’ Test
Steel’s test
Tamhane-Dunnett
Tamhane-Dunnett test
the method Davies
Vogel-Bonner Medium E
Welch’s test
Welch t-test
Whitney test
Wilcoxon and Ansari-Bradley statistics
Williams multiple sequential t-test
Williams' test
Williams t-test
William test
Winkler titration technique,

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrates.txt → redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/vertebrate.txt
View File

@ -1,4 +1,3 @@
a. sylvaticus
african clawed frog
agalychnis callidryas
albino rat
@ -15,6 +14,7 @@ apodemus flavicollis
apodemus syl vaticus
apodemus sylvaticus
arvicola terrestris
a. sylvaticus
avian
bank vole
bird
@ -23,6 +23,7 @@ bluegill
bluegill sunfish
bobwhite
bobwhite quail
Bovine
brachydanio rerio
brown hare
bufo americanus
@ -52,10 +53,12 @@ crocidura russula
crucian carp
cyprinodon variegatus
cyprinus carpio
(Danio rerio
dog
dogs
duck
ducks
Environmental & Molecular Mutagenesis
european brown hare
european rabbit
fathead minnow
@ -121,14 +124,13 @@ myodes glareolus
northern bobwhite
o. cuniculus
o. mykiss
o. tshawytscha
oncorhynchus
oncorhynchus mykiss
oncorhynchus tshawytscha
oryctolagus cuniculus
oryzias melastigma
oryzias melastigma larvae
p. promelas
o. tshawytscha
pagrus major
palumbus
pig
@ -139,10 +141,10 @@ pimephales promela
pimephales promelas
poecilia reticulata
poultry
p. promelas
pseudacris
pseudacris triseriata
quail
r. catesbeiana
rabbit
rabbits
rainbow trout
@ -152,6 +154,7 @@ rana limnocharis
rana pipiens
rat
rats
r. catesbeiana
reptile
reptiles
ricefish
@ -164,10 +167,12 @@ sheepshead minnow
sheepshead minnows
shrew
shrews
Singh
sorex araneus
spea multiplicata
spotted march frog
tadpoles
Taeniopygia guttata
terrestrial vertrebrates
toad
treefrog
@ -187,5 +192,6 @@ wood pigeon
xenopus laevis
xenpous leavis
yellow-necked mouse
Zebra Finch
zebra fish
zebrafish
zebrafish

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -9,7 +9,7 @@ rule "1: Redacted because Section contains Vertebrate"
when
Section(matchesType("vertebrate"))
then
section.redact("name", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -18,7 +18,7 @@ rule "2: Not Redacted because Section contains no Vertebrate"
when
Section(!matchesType("vertebrate"))
then
section.redactNot("name", 2, "No Vertebrate found");
section.redactNot("author", 2, "No Vertebrate found");
section.redactNot("address", 2, "No Vertebrate found");
end
@ -27,7 +27,7 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
when
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
then
section.redactNot("name", 3, "Vertebrate and No Redaction Indicator found");
section.redactNot("author", 3, "Vertebrate and No Redaction Indicator found");
section.redactNot("address", 3, "Vertebrate and No Redaction Indicator found");
end
@ -36,7 +36,7 @@ rule "4: Do not redact Names and Addresses if no redaction Indicator is containe
when
Section(matchesType("vertebrate"), matchesType("published_information"))
then
section.redactNot("name", 4, "Vertebrate and Published Information found");
section.redactNot("author", 4, "Vertebrate and Published Information found");
section.redactNot("address", 4, "Vertebrate and Published Information found");
end
@ -45,7 +45,7 @@ rule "5: Redact Names and Addresses if no_redaction_indicator and redaction_indi
when
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
then
section.redact("name", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("author", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("address", 5, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -97,7 +97,7 @@ rule "8: Not redacted because Vertebrate Study = N"
when
Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
then
section.redactNotCell("Author(s)", 8, "name", "Not redacted because row is not a vertebrate study");
section.redactNotCell("Author(s)", 8, "author", "Not redacted because row is not a vertebrate study");
section.redactNot("address", 8, "Not redacted because row is not a vertebrate study");
section.highlightCell("Vertebrate study Y/N", 8, "hint_only");
end
@ -107,7 +107,7 @@ rule "9: Redact if must redact entry is found"
when
Section(matchesType("must_redact"))
then
section.redact("name", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("author", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("address", 9, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -116,7 +116,7 @@ rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate
when
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
then
section.redactCell("Author(s)", 10, "name", "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redactCell("Author(s)", 10, "author", "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.redact("address", 10, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
end