Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Pull request #46: Add rule redacting sponsor companies if preceded by prefix

Browse Source 
Merge in RED/redaction-service from RED-301 to master

* commit '6a8f3665198ec2d7452ed0dac43b9a7d220c8bc5':
  Remove entries from must_redact dict, add test and refactor rules
  Add rule redacting sponsor companies if preceded by prefix
This commit is contained in:
Thierry Goeckel 2020-10-01 11:44:41 +02:00
commit 589458a112
6 changed files with 74 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java
View File

@ -44,7 +44,7 @@ public class Section {
}
public boolean contains(String type) {
public boolean matchesType(String type) {
return entities.stream().anyMatch(entity -> entity.getType().equals(type));
}
@ -80,6 +80,18 @@ public class Section {
}
public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason) {
entities.forEach(entity -> {
if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) {
entity.setRedaction(true);
entity.setMatchedRule(ruleNumber);
entity.setRedactionReason(reason);
}
});
}
public void redactLineAfter(String start, String asType, int ruleNumber, String reason) {
String[] values = StringUtils.substringsBetween(text, start, "\n");
@ -141,7 +153,8 @@ public class Section {
if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText
.charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) {
found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber));
found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex,
headline, sectionNumber));
}
} while (startIndex > -1);
@ -197,7 +210,8 @@ public class Section {
if (value == null) {
log.warn("Could not find any data for {}.", cellHeader);
} else {
Entity entity = new Entity(value.toString(), type, value.getRowSpanStart(), value.getRowSpanStart() + value.toString()
Entity entity = new Entity(value.toString(), type, value.getRowSpanStart(),
value.getRowSpanStart() + value.toString()
.length(), headline, sectionNumber);
entity.setRedaction(redact);
entity.setMatchedRule(ruleNumber);
@ -206,7 +220,7 @@ public class Section {
.getSequences()); // Make sure no other cells with same content are highlighted
// HashSet keeps the older value, but we want the new only.
if(entities.contains(entity)){
if (entities.contains(entity)) {
entities.remove(entity);
}
entities.add(entity);

34
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -63,6 +63,7 @@ public class RedactionIntegrationTest {
private static final String VERTEBRATES_CODE = "vertebrate";
private static final String ADDRESS_CODE = "address";
private static final String NAME_CODE = "name";
private static final String SPONSOR = "sponsor";
private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator";
private static final String REDACTION_INDICATOR = "redaction_indicator";
private static final String HINT_ONLY = "hint_only";
@ -117,6 +118,7 @@ public class RedactionIntegrationTest {
when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE));
when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE));
when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE));
when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR));
when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR));
when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY));
@ -132,6 +134,11 @@ public class RedactionIntegrationTest {
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/sponsor_companies.txt")
.stream()
.map(this::cleanDictionaryEntry)
.collect(Collectors.toSet()));
dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>())
.addAll(ResourceLoader.load("dictionaries/vertebrates.txt")
.stream()
@ -176,6 +183,7 @@ public class RedactionIntegrationTest {
typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0});
typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1});
typeColorMap.put(NAME_CODE, new float[]{1, 1, 0});
typeColorMap.put(SPONSOR, new float[]{.5f, .5f, .5f});
typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f});
typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f});
typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f});
@ -184,6 +192,7 @@ public class RedactionIntegrationTest {
hintTypeMap.put(VERTEBRATES_CODE, true);
hintTypeMap.put(ADDRESS_CODE, false);
hintTypeMap.put(NAME_CODE, false);
hintTypeMap.put(SPONSOR, false);
hintTypeMap.put(NO_REDACTION_INDICATOR, true);
hintTypeMap.put(REDACTION_INDICATOR, true);
hintTypeMap.put(HINT_ONLY, true);
@ -192,6 +201,7 @@ public class RedactionIntegrationTest {
caseInSensitiveMap.put(VERTEBRATES_CODE, true);
caseInSensitiveMap.put(ADDRESS_CODE, false);
caseInSensitiveMap.put(NAME_CODE, false);
caseInSensitiveMap.put(SPONSOR, false);
caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true);
caseInSensitiveMap.put(REDACTION_INDICATOR, true);
caseInSensitiveMap.put(HINT_ONLY, true);
@ -435,13 +445,35 @@ public class RedactionIntegrationTest {
RedactionResult result = redactionController.redact(request);
result.getRedactionLog().getRedactionLogEntry().forEach(entry -> {
if(!entry.isHint()){
if (!entry.isHint()) {
assertThat(entry.getReason()).isEqualTo("Not redacted because row is not a vertebrate study");
}
});
}
@Test
public void sponsorCompanyTest() throws IOException {
long start = System.currentTimeMillis();
ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf");
RedactionRequest request = RedactionRequest.builder()
.document(IOUtils.toByteArray(pdfFileResource.getInputStream()))
.build();
request.setFlatRedaction(false);
RedactionResult result = redactionController.redact(request);
try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) {
fileOutputStream.write(result.getDocument());
}
long end = System.currentTimeMillis();
System.out.println("duration: " + (end - start));
System.out.println("numberOfPages: " + result.getNumberOfPages());
}
private static String loadFromClassPath(String path) {

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt
View File

@ -1,3 +1,2 @@
Batches Produced at
CTL
determination of residues

7
redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor_companies.txt Normal file
View File

@ -0,0 +1,7 @@
Monthey Syngenta Crop Protection AG, Basel, Switzerland
Syngenta Crop Protection, Monthey, Switzerland
Fine Organics Limited, Middlesbrough, United Kingdom
Syngenta Monthey Switzerland
Hunan Haili Chemical Industry Co., Ltd., Hunan, China
Syngenta, Switzerland
Syngenta Nantong, China

24
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -7,7 +7,7 @@ global Section section
rule "1: Redacted because Section contains Vertebrate"
when
eval(section.contains("vertebrate")==true);
Section(matchesType("vertebrate"))
then
section.redact("name", 1, "Redacted because Section contains Vertebrate");
section.redact("address", 1, "Redacted because Section contains Vertebrate");
@ -16,7 +16,7 @@ rule "1: Redacted because Section contains Vertebrate"
rule "2: Not Redacted because Section contains no Vertebrate"
when
eval(section.contains("vertebrate")==false);
Section(matchesType("vertebrate"))
then
section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate");
section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate");
@ -25,7 +25,7 @@ rule "2: Not Redacted because Section contains no Vertebrate"
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
when
eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true);
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
then
section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator");
section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator");
@ -34,7 +34,7 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
when
eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true);
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
then
section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator");
@ -43,15 +43,16 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi
rule "5: Do not redact in guideline sections"
when
eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance"));
Section(headlineContainsWord("guideline") || headlineContainsWord("Guidance"))
then
section.redactNot("name", 5, "Section is a guideline section.");
section.redactNot("address", 5, "Section is a guideline section.");
end
rule "6: Redact contact information if applicant is found"
when
eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant") || section.headlineContainsWord("Primary contact") || section.headlineContainsWord("Alternative contact"));
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact"))
then
section.redactLineAfter("Name:", "address", 6, "Applicant information was found");
section.redactBetween("Address:", "Contact", "address", 6, "Applicant information was found");
@ -78,7 +79,7 @@ rule "6: Redact contact information if applicant is found"
rule "7: Redact contact information if Producer is found"
when
eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance"));
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
then
section.redactLineAfter("Name:", "address", 7, "Producer was found");
section.redactBetween("Address:", "Contact", "address", 7, "Producer was found");
@ -109,7 +110,7 @@ rule "8: Not redacted because Vertebrate Study = N"
rule "9: Redact if must redact entry is found"
when
eval(section.contains("must_redact")==true);
Section(matchesType("must_redact"))
then
section.redact("name", 9, "must_redact entry was found.");
section.redact("address", 9, "must_redact entry was found.");
@ -124,3 +125,10 @@ rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate
section.redact("address", 10, "Redacted because row is a vertebrate study");
section.highlightCell("Vertebrate study Y/N", 10, "must_redact");
end
rule "11: Redact sponsor company"
when
Section(text.toLowerCase().contains("batches produced at"))
then
section.redactIfPrecededBy("batches produced at", "sponsor", 11, "Redacted because it represents a sponsor company");
end

BIN
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/sponsor_companies.pdf Normal file
View File

Binary file not shown.