Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #103: RED-934: Added rule to redact purity and et al Authors will be redacted where they are found

Browse Source 
Merge in RED/redaction-service from RED-934 to master

* commit '33795527fdc4fd82f6ad26ec5720a539620b1293':
  RED-934: Added rule to redact purity and et al Authors will be redacted where they are found
This commit is contained in:
Dominique Eiflaender 2021-01-20 13:30:16 +01:00
commit 375d84b236
5 changed files with 55 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -201,6 +201,24 @@ public class Section {
}
public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
String reason, String legalBasis) {
Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
Matcher matcher = compiledPattern.matcher(searchText);
while (matcher.find()) {
String match = matcher.group(group);
if (StringUtils.isNotBlank(match) && match.length() >= 3) {
Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
addNewerToEntities(found);
localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
}
}
}
public void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
String reason, String legalBasis) {

13
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -75,6 +75,7 @@ public class RedactionIntegrationTest {
private static final String MUST_REDACT = "must_redact";
private static final String PUBLISHED_INFORMATION = "published_information";
private static final String TEST_METHOD = "test_method";
private static final String PURITY = "purity";
private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@ -147,6 +148,7 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
}
@ -223,6 +225,11 @@ public class RedactionIntegrationTest {
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/purity.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
}
@ -248,6 +255,7 @@ public class RedactionIntegrationTest {
typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
typeColorMap.put(FALSE_POSITIVE, "#ffffff");
typeColorMap.put(PURITY, "#ffe187");
hintTypeMap.put(VERTEBRATE, true);
hintTypeMap.put(ADDRESS, false);
@ -263,6 +271,7 @@ public class RedactionIntegrationTest {
hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
hintTypeMap.put(FALSE_POSITIVE, true);
hintTypeMap.put(PURITY, false);
caseInSensitiveMap.put(VERTEBRATE, true);
caseInSensitiveMap.put(ADDRESS, false);
@ -278,6 +287,7 @@ public class RedactionIntegrationTest {
caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
caseInSensitiveMap.put(FALSE_POSITIVE, false);
caseInSensitiveMap.put(PURITY, false);
recommendationTypeMap.put(VERTEBRATE, false);
recommendationTypeMap.put(ADDRESS, false);
@ -293,6 +303,7 @@ public class RedactionIntegrationTest {
recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
recommendationTypeMap.put(FALSE_POSITIVE, false);
recommendationTypeMap.put(PURITY, false);
colors.setDefaultColor("#acfc00");
colors.setNotRedacted("#cccccc");
@ -390,7 +401,7 @@ public class RedactionIntegrationTest {
System.out.println("redactionTest");
long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_08_Volume_3CA_B-6_2018-09-06.pdf");
RedactionRequest request = RedactionRequest.builder()
.ruleSetId(TEST_RULESET_ID)

0
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/purity.txt Normal file
View File

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl
View File

@ -133,11 +133,11 @@ rule "12: Redact if CTL/* or BL/* was found"
end
rule "13: Add recommendation for et al. author"
rule "13: Redact and add recommendation for et al. author"
when
Section(searchText.contains("et al."))
Section(searchText.contains("et al"))
then
section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author");
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 13, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -286,4 +286,14 @@ rule "24: Redact On behalf of Syngenta Ltd.:"
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
then
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 24, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "25: Redact Purity"
when
Section(searchText.contains("purity"))
then
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
end

15
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -134,11 +134,11 @@ rule "11: Redact if CTL/* or BL/* was found"
end
rule "12: Add recommendation for et al. author"
rule "12: Redact and add recommendation for et al. author"
when
Section(searchText.contains("et al."))
Section(searchText.contains("et al"))
then
section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author");
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
end
@ -286,4 +286,13 @@ rule "22: Redact On behalf of Syngenta Ltd.:"
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
then
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
end
// --------------------------------------- other rules -------------------------------------------------------------------
rule "25: Redact Purity"
when
Section(searchText.contains("purity"))
then
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
end