diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index 8b26da67..b05db9c0 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -201,6 +201,24 @@ public class Section { } + public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber, + String reason, String legalBasis) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(searchText); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match) && match.length() >= 3) { + Set found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis); + addNewerToEntities(found); + localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match); + } + } + } + + public void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere, String reason, String legalBasis) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 4bb39477..44eca77c 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -75,6 +75,7 @@ public class RedactionIntegrationTest { private static final String MUST_REDACT = "must_redact"; private static final String PUBLISHED_INFORMATION = "published_information"; private static final String TEST_METHOD = "test_method"; + private static final String PURITY = "purity"; private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author"; private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address"; @@ -147,6 +148,7 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR)); when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS)); when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE)); + when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY)); when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors); } @@ -223,6 +225,11 @@ public class RedactionIntegrationTest { .stream() .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); + dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/purity.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); } @@ -248,6 +255,7 @@ public class RedactionIntegrationTest { typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c"); typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c"); typeColorMap.put(FALSE_POSITIVE, "#ffffff"); + typeColorMap.put(PURITY, "#ffe187"); hintTypeMap.put(VERTEBRATE, true); hintTypeMap.put(ADDRESS, false); @@ -263,6 +271,7 @@ public class RedactionIntegrationTest { hintTypeMap.put(RECOMMENDATION_AUTHOR, false); hintTypeMap.put(RECOMMENDATION_ADDRESS, false); hintTypeMap.put(FALSE_POSITIVE, true); + hintTypeMap.put(PURITY, false); caseInSensitiveMap.put(VERTEBRATE, true); caseInSensitiveMap.put(ADDRESS, false); @@ -278,6 +287,7 @@ public class RedactionIntegrationTest { caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false); caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false); caseInSensitiveMap.put(FALSE_POSITIVE, false); + caseInSensitiveMap.put(PURITY, false); recommendationTypeMap.put(VERTEBRATE, false); recommendationTypeMap.put(ADDRESS, false); @@ -293,6 +303,7 @@ public class RedactionIntegrationTest { recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true); recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true); recommendationTypeMap.put(FALSE_POSITIVE, false); + recommendationTypeMap.put(PURITY, false); colors.setDefaultColor("#acfc00"); colors.setNotRedacted("#cccccc"); @@ -390,7 +401,7 @@ public class RedactionIntegrationTest { System.out.println("redactionTest"); long start = System.currentTimeMillis(); - ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf"); + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_08_Volume_3CA_B-6_2018-09-06.pdf"); RedactionRequest request = RedactionRequest.builder() .ruleSetId(TEST_RULESET_ID) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/purity.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/purity.txt new file mode 100644 index 00000000..e69de29b diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl index ff3140d3..813fb1a9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl @@ -133,11 +133,11 @@ rule "12: Redact if CTL/* or BL/* was found" end -rule "13: Add recommendation for et al. author" +rule "13: Redact and add recommendation for et al. author" when - Section(searchText.contains("et al.")) + Section(searchText.contains("et al")) then - section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 13, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); end @@ -286,4 +286,14 @@ rule "24: Redact On behalf of Syngenta Ltd.:" Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title")) then section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 24, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + end + + +// --------------------------------------- other rules ------------------------------------------------------------------- + +rule "25: Redact Purity" + when + Section(searchText.contains("purity")) + then + section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 852e57ae..fe8d75fb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -134,11 +134,11 @@ rule "11: Redact if CTL/* or BL/* was found" end -rule "12: Add recommendation for et al. author" +rule "12: Redact and add recommendation for et al. author" when - Section(searchText.contains("et al.")) + Section(searchText.contains("et al")) then - section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author"); + section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)"); end @@ -286,4 +286,13 @@ rule "22: Redact On behalf of Syngenta Ltd.:" Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title")) then section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)"); + end + +// --------------------------------------- other rules ------------------------------------------------------------------- + +rule "25: Redact Purity" + when + Section(searchText.contains("purity")) + then + section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"); end \ No newline at end of file