RED-934: Added rule to redact purity and et al Authors will be redacted where they are found

redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java

@@ -201,6 +201,24 @@ public class Section {
     }
 
 
+    public void redactAndRecommendByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType, int ruleNumber,
+                              String reason, String legalBasis) {
+
+        Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive);
+
+        Matcher matcher = compiledPattern.matcher(searchText);
+
+        while (matcher.find()) {
+            String match = matcher.group(group);
+            if (StringUtils.isNotBlank(match) && match.length() >= 3) {
+                Set<Entity> found = findEntities(match.trim(), asType, false, true, ruleNumber, reason, legalBasis);
+                addNewerToEntities(found);
+                localDictionaryAdds.computeIfAbsent(RECOMMENDATION_PREFIX + asType, (x) -> new HashSet<>()).add(match);
+            }
+        }
+    }
+
+
     public void redactBetween(String start, String stop, String asType, int ruleNumber, boolean redactEverywhere,
                               String reason, String legalBasis) {
 

redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java

@@ -75,6 +75,7 @@ public class RedactionIntegrationTest {
     private static final String MUST_REDACT = "must_redact";
     private static final String PUBLISHED_INFORMATION = "published_information";
     private static final String TEST_METHOD = "test_method";
+    private static final String PURITY = "purity";
 
     private static final String RECOMMENDATION_AUTHOR = "recommendation_CBI_author";
     private static final String RECOMMENDATION_ADDRESS = "recommendation_CBI_address";
@@ -147,6 +148,7 @@ public class RedactionIntegrationTest {
         when(dictionaryClient.getDictionaryForType(RECOMMENDATION_AUTHOR, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_AUTHOR));
         when(dictionaryClient.getDictionaryForType(RECOMMENDATION_ADDRESS, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(RECOMMENDATION_ADDRESS));
         when(dictionaryClient.getDictionaryForType(FALSE_POSITIVE, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(FALSE_POSITIVE));
+        when(dictionaryClient.getDictionaryForType(PURITY, TEST_RULESET_ID)).thenReturn(getDictionaryResponse(PURITY));
         when(dictionaryClient.getColors(TEST_RULESET_ID)).thenReturn(colors);
     }
 
@@ -223,6 +225,11 @@ public class RedactionIntegrationTest {
                         .stream()
                         .map(this::cleanDictionaryEntry)
                         .collect(Collectors.toSet()));
+        dictionary.computeIfAbsent(PURITY, v -> new ArrayList<>())
+                .addAll(ResourceLoader.load("dictionaries/purity.txt")
+                        .stream()
+                        .map(this::cleanDictionaryEntry)
+                        .collect(Collectors.toSet()));
     }
 
 
@@ -248,6 +255,7 @@ public class RedactionIntegrationTest {
         typeColorMap.put(RECOMMENDATION_AUTHOR, "#8df06c");
         typeColorMap.put(RECOMMENDATION_ADDRESS, "#8df06c");
         typeColorMap.put(FALSE_POSITIVE, "#ffffff");
+        typeColorMap.put(PURITY, "#ffe187");
 
         hintTypeMap.put(VERTEBRATE, true);
         hintTypeMap.put(ADDRESS, false);
@@ -263,6 +271,7 @@ public class RedactionIntegrationTest {
         hintTypeMap.put(RECOMMENDATION_AUTHOR, false);
         hintTypeMap.put(RECOMMENDATION_ADDRESS, false);
         hintTypeMap.put(FALSE_POSITIVE, true);
+        hintTypeMap.put(PURITY, false);
 
         caseInSensitiveMap.put(VERTEBRATE, true);
         caseInSensitiveMap.put(ADDRESS, false);
@@ -278,6 +287,7 @@ public class RedactionIntegrationTest {
         caseInSensitiveMap.put(RECOMMENDATION_AUTHOR, false);
         caseInSensitiveMap.put(RECOMMENDATION_ADDRESS, false);
         caseInSensitiveMap.put(FALSE_POSITIVE, false);
+        caseInSensitiveMap.put(PURITY, false);
 
         recommendationTypeMap.put(VERTEBRATE, false);
         recommendationTypeMap.put(ADDRESS, false);
@@ -293,6 +303,7 @@ public class RedactionIntegrationTest {
         recommendationTypeMap.put(RECOMMENDATION_AUTHOR, true);
         recommendationTypeMap.put(RECOMMENDATION_ADDRESS, true);
         recommendationTypeMap.put(FALSE_POSITIVE, false);
+        recommendationTypeMap.put(PURITY, false);
 
         colors.setDefaultColor("#acfc00");
         colors.setNotRedacted("#cccccc");
@@ -390,7 +401,7 @@ public class RedactionIntegrationTest {
 
         System.out.println("redactionTest");
         long start = System.currentTimeMillis();
-        ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/Applicant Producer Table.pdf");
+        ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_08_Volume_3CA_B-6_2018-09-06.pdf");
 
         RedactionRequest request = RedactionRequest.builder()
                 .ruleSetId(TEST_RULESET_ID)

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/allAuthors.drl

@@ -133,11 +133,11 @@ rule "12: Redact if CTL/* or BL/*  was found"
     end
 
 
-rule "13: Add recommendation for et al. author"
+rule "13: Redact and add recommendation for et al. author"
     when
-        Section(searchText.contains("et al."))
+        Section(searchText.contains("et al"))
     then
-		section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 13, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
 
 
@@ -287,3 +287,13 @@ rule "24: Redact On behalf of Syngenta Ltd.:"
     then
         section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 24, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
     end
+
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "25: Redact Purity"
+    when
+        Section(searchText.contains("purity"))
+    then
+	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+    end

redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl

@@ -134,11 +134,11 @@ rule "11: Redact if CTL/* or BL/*  was found"
     end
 
 
-rule "12: Add recommendation for et al. author"
+rule "12: Redact and add recommendation for et al. author"
     when
-        Section(searchText.contains("et al."))
+        Section(searchText.contains("et al"))
     then
-		section.addRecommendationByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]\\.?)?( [A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author");
+		section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
     end
 
 
@@ -287,3 +287,12 @@ rule "22: Redact On behalf of Syngenta Ltd.:"
     then
         section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
     end
+
+// --------------------------------------- other rules -------------------------------------------------------------------
+
+rule "25: Redact Purity"
+    when
+        Section(searchText.contains("purity"))
+    then
+	    section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
+    end