diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index dc14e5a8..68e7140b 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -44,7 +44,7 @@ public class Section { } - public boolean contains(String type) { + public boolean matchesType(String type) { return entities.stream().anyMatch(entity -> entity.getType().equals(type)); } @@ -80,6 +80,18 @@ public class Section { } + public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason) { + + entities.forEach(entity -> { + if (entity.getType().equals(type) && searchText.indexOf(prefix + entity.getWord()) != 1) { + entity.setRedaction(true); + entity.setMatchedRule(ruleNumber); + entity.setRedactionReason(reason); + } + }); + } + + public void redactLineAfter(String start, String asType, int ruleNumber, String reason) { String[] values = StringUtils.substringsBetween(text, start, "\n"); @@ -141,7 +153,8 @@ public class Section { if (startIndex > -1 && (startIndex == 0 || Character.isWhitespace(searchText.charAt(startIndex - 1)) || isSeparator(searchText .charAt(startIndex - 1))) && (stopIndex == searchText.length() || isSeparator(searchText.charAt(stopIndex)))) { - found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, headline, sectionNumber)); + found.add(new Entity(searchText.substring(startIndex, stopIndex), asType, startIndex, stopIndex, + headline, sectionNumber)); } } while (startIndex > -1); @@ -197,7 +210,8 @@ public class Section { if (value == null) { log.warn("Could not find any data for {}.", cellHeader); } else { - Entity entity = new Entity(value.toString(), type, value.getRowSpanStart(), value.getRowSpanStart() + value.toString() + Entity entity = new Entity(value.toString(), type, value.getRowSpanStart(), + value.getRowSpanStart() + value.toString() .length(), headline, sectionNumber); entity.setRedaction(redact); entity.setMatchedRule(ruleNumber); @@ -206,7 +220,7 @@ public class Section { .getSequences()); // Make sure no other cells with same content are highlighted // HashSet keeps the older value, but we want the new only. - if(entities.contains(entity)){ + if (entities.contains(entity)) { entities.remove(entity); } entities.add(entity); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java index 4a85444d..5c942080 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java @@ -63,6 +63,7 @@ public class RedactionIntegrationTest { private static final String VERTEBRATES_CODE = "vertebrate"; private static final String ADDRESS_CODE = "address"; private static final String NAME_CODE = "name"; + private static final String SPONSOR = "sponsor"; private static final String NO_REDACTION_INDICATOR = "no_redaction_indicator"; private static final String REDACTION_INDICATOR = "redaction_indicator"; private static final String HINT_ONLY = "hint_only"; @@ -117,6 +118,7 @@ public class RedactionIntegrationTest { when(dictionaryClient.getDictionaryForType(VERTEBRATES_CODE)).thenReturn(getDictionaryResponse(VERTEBRATES_CODE)); when(dictionaryClient.getDictionaryForType(ADDRESS_CODE)).thenReturn(getDictionaryResponse(ADDRESS_CODE)); when(dictionaryClient.getDictionaryForType(NAME_CODE)).thenReturn(getDictionaryResponse(NAME_CODE)); + when(dictionaryClient.getDictionaryForType(SPONSOR)).thenReturn(getDictionaryResponse(SPONSOR)); when(dictionaryClient.getDictionaryForType(NO_REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(NO_REDACTION_INDICATOR)); when(dictionaryClient.getDictionaryForType(REDACTION_INDICATOR)).thenReturn(getDictionaryResponse(REDACTION_INDICATOR)); when(dictionaryClient.getDictionaryForType(HINT_ONLY)).thenReturn(getDictionaryResponse(HINT_ONLY)); @@ -132,6 +134,11 @@ public class RedactionIntegrationTest { .stream() .map(this::cleanDictionaryEntry) .collect(Collectors.toSet())); + dictionary.computeIfAbsent(SPONSOR, v -> new ArrayList<>()) + .addAll(ResourceLoader.load("dictionaries/sponsor_companies.txt") + .stream() + .map(this::cleanDictionaryEntry) + .collect(Collectors.toSet())); dictionary.computeIfAbsent(VERTEBRATES_CODE, v -> new ArrayList<>()) .addAll(ResourceLoader.load("dictionaries/vertebrates.txt") .stream() @@ -176,6 +183,7 @@ public class RedactionIntegrationTest { typeColorMap.put(VERTEBRATES_CODE, new float[]{0, 1, 0}); typeColorMap.put(ADDRESS_CODE, new float[]{0, 1, 1}); typeColorMap.put(NAME_CODE, new float[]{1, 1, 0}); + typeColorMap.put(SPONSOR, new float[]{.5f, .5f, .5f}); typeColorMap.put(NO_REDACTION_INDICATOR, new float[]{0.8f, 0, 0.8f}); typeColorMap.put(REDACTION_INDICATOR, new float[]{1, 0.502f, 0.1f}); typeColorMap.put(HINT_ONLY, new float[]{0.8f, 1, 0.8f}); @@ -184,6 +192,7 @@ public class RedactionIntegrationTest { hintTypeMap.put(VERTEBRATES_CODE, true); hintTypeMap.put(ADDRESS_CODE, false); hintTypeMap.put(NAME_CODE, false); + hintTypeMap.put(SPONSOR, false); hintTypeMap.put(NO_REDACTION_INDICATOR, true); hintTypeMap.put(REDACTION_INDICATOR, true); hintTypeMap.put(HINT_ONLY, true); @@ -192,6 +201,7 @@ public class RedactionIntegrationTest { caseInSensitiveMap.put(VERTEBRATES_CODE, true); caseInSensitiveMap.put(ADDRESS_CODE, false); caseInSensitiveMap.put(NAME_CODE, false); + caseInSensitiveMap.put(SPONSOR, false); caseInSensitiveMap.put(NO_REDACTION_INDICATOR, true); caseInSensitiveMap.put(REDACTION_INDICATOR, true); caseInSensitiveMap.put(HINT_ONLY, true); @@ -435,13 +445,35 @@ public class RedactionIntegrationTest { RedactionResult result = redactionController.redact(request); result.getRedactionLog().getRedactionLogEntry().forEach(entry -> { - if(!entry.isHint()){ + if (!entry.isHint()) { assertThat(entry.getReason()).isEqualTo("Not redacted because row is not a vertebrate study"); } }); } + @Test + public void sponsorCompanyTest() throws IOException { + + long start = System.currentTimeMillis(); + ClassPathResource pdfFileResource = new ClassPathResource("files/Minimal Examples/sponsor_companies.pdf"); + + RedactionRequest request = RedactionRequest.builder() + .document(IOUtils.toByteArray(pdfFileResource.getInputStream())) + .build(); + request.setFlatRedaction(false); + + RedactionResult result = redactionController.redact(request); + + try (FileOutputStream fileOutputStream = new FileOutputStream("/tmp/Redacted.pdf")) { + fileOutputStream.write(result.getDocument()); + } + long end = System.currentTimeMillis(); + + System.out.println("duration: " + (end - start)); + System.out.println("numberOfPages: " + result.getNumberOfPages()); + } + private static String loadFromClassPath(String path) { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt index 34387b57..0a4416d4 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/must_redact.txt @@ -1,3 +1,2 @@ -Batches Produced at CTL determination of residues \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor_companies.txt b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor_companies.txt new file mode 100644 index 00000000..2b1652fa --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/dictionaries/sponsor_companies.txt @@ -0,0 +1,7 @@ +Monthey Syngenta Crop Protection AG, Basel, Switzerland +Syngenta Crop Protection, Monthey, Switzerland +Fine Organics Limited, Middlesbrough, United Kingdom +Syngenta Monthey Switzerland +Hunan Haili Chemical Industry Co., Ltd., Hunan, China +Syngenta, Switzerland +Syngenta Nantong, China \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl index 2490aad2..fc0cc47f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl @@ -7,7 +7,7 @@ global Section section rule "1: Redacted because Section contains Vertebrate" when - eval(section.contains("vertebrate")==true); + Section(matchesType("vertebrate")) then section.redact("name", 1, "Redacted because Section contains Vertebrate"); section.redact("address", 1, "Redacted because Section contains Vertebrate"); @@ -16,7 +16,7 @@ rule "1: Redacted because Section contains Vertebrate" rule "2: Not Redacted because Section contains no Vertebrate" when - eval(section.contains("vertebrate")==false); + Section(matchesType("vertebrate")) then section.redactNot("name", 2, "Not Redacted because Section contains no Vertebrate"); section.redactNot("address", 2, "Not Redacted because Section contains no Vertebrate"); @@ -25,7 +25,7 @@ rule "2: Not Redacted because Section contains no Vertebrate" rule "3: Do not redact Names and Addresses if no redaction Indicator is contained" when - eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true); + Section(matchesType("vertebrate"), matchesType("no_redaction_indicator")) then section.redactNot("name", 3, "Vertebrate was found, but also a no redaction indicator"); section.redactNot("address", 3, "Vertebrate was found, but also a no redaction indicator"); @@ -34,7 +34,7 @@ rule "3: Do not redact Names and Addresses if no redaction Indicator is containe rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained" when - eval(section.contains("vertebrate")==true && section.contains("no_redaction_indicator")==true && section.contains("redaction_indicator")==true); + Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator")) then section.redact("name", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator"); section.redact("address", 4, "Vertebrate was found and no_redaction_indicator and redaction_indicator"); @@ -43,15 +43,16 @@ rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indi rule "5: Do not redact in guideline sections" when - eval(section.headlineContainsWord("guideline") || section.headlineContainsWord("Guidance")); + Section(headlineContainsWord("guideline") || headlineContainsWord("Guidance")) then section.redactNot("name", 5, "Section is a guideline section."); section.redactNot("address", 5, "Section is a guideline section."); end + rule "6: Redact contact information if applicant is found" when - eval(section.headlineContainsWord("applicant") || section.getText().contains("Applicant") || section.headlineContainsWord("Primary contact") || section.headlineContainsWord("Alternative contact")); + Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact")) then section.redactLineAfter("Name:", "address", 6, "Applicant information was found"); section.redactBetween("Address:", "Contact", "address", 6, "Applicant information was found"); @@ -78,7 +79,7 @@ rule "6: Redact contact information if applicant is found" rule "7: Redact contact information if Producer is found" when - eval(section.getText().toLowerCase().contains("producer of the plant protection") || section.getText().toLowerCase().contains("producer of the active substance") || section.getText().contains("Manufacturer of the active substance") || section.getText().contains("Manufacturer:") || section.getText().contains("Producer or producers of the active substance")); + Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance")) then section.redactLineAfter("Name:", "address", 7, "Producer was found"); section.redactBetween("Address:", "Contact", "address", 7, "Producer was found"); @@ -109,7 +110,7 @@ rule "8: Not redacted because Vertebrate Study = N" rule "9: Redact if must redact entry is found" when - eval(section.contains("must_redact")==true); + Section(matchesType("must_redact")) then section.redact("name", 9, "must_redact entry was found."); section.redact("address", 9, "must_redact entry was found."); @@ -124,3 +125,10 @@ rule "10: Redact Authors and Addresses in Reference Table if it is a Vertebrate section.redact("address", 10, "Redacted because row is a vertebrate study"); section.highlightCell("Vertebrate study Y/N", 10, "must_redact"); end + +rule "11: Redact sponsor company" + when + Section(text.toLowerCase().contains("batches produced at")) + then + section.redactIfPrecededBy("batches produced at", "sponsor", 11, "Redacted because it represents a sponsor company"); + end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/sponsor_companies.pdf b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/sponsor_companies.pdf new file mode 100644 index 00000000..45b9f39a Binary files /dev/null and b/redaction-service-v1/redaction-service-server-v1/src/test/resources/files/Minimal Examples/sponsor_companies.pdf differ