From 7f7de6ce2a65820f38d90a9e14dbfdd0d3ccadb6 Mon Sep 17 00:00:00 2001 From: lmaldacker Date: Thu, 11 Feb 2021 13:48:43 +0100 Subject: [PATCH] purity --- .../v1/server/redaction/model/Section.java | 22 ++++++++++- .../service/EntityRedactionServiceTest.java | 38 +++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java index f53220ad..d6123ef6 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/model/Section.java @@ -140,7 +140,7 @@ public class Section { } - public void dismissByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group) { + public void expandToHintAnnotationByRegEx(String type, String pattern, boolean patternCaseInsensitive, int group, String asType) { Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); @@ -156,7 +156,7 @@ public class Section { while (matcher.find()) { String match = matcher.group(group); if (StringUtils.isNotBlank(match)) { - expanded.addAll(findEntities(entity.getWord() + match, type, false, entity.isRedaction(), 0, null, null)); + expanded.addAll(findEntities(entity.getWord() + match, asType, false, false, 0, null, null)); } } } @@ -166,6 +166,24 @@ public class Section { } + + + public void addHintAnnotationByRegEx(String pattern, boolean patternCaseInsensitive, int group, String asType) { + + Pattern compiledPattern = Patterns.getCompiledPattern(pattern, patternCaseInsensitive); + + Matcher matcher = compiledPattern.matcher(searchText); + + while (matcher.find()) { + String match = matcher.group(group); + if (StringUtils.isNotBlank(match)) { + Set found = findEntities(match.trim(), asType, false, false, 0, null, null); + EntitySearchUtils.addEntitiesWithHigherRank(entities, found, dictionary); + } + } + } + + public void redactIfPrecededBy(String prefix, String type, int ruleNumber, String reason, String legalBasis) { entities.forEach(entity -> { diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java index fb2fae12..67b69070 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/redaction/service/EntityRedactionServiceTest.java @@ -119,6 +119,44 @@ public class EntityRedactionServiceTest { } + @Test + public void testGenitiveDismissal() throws IOException { + + ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); + when(dictionaryClient.getVersion(TEST_RULESET_ID)).thenReturn(DICTIONARY_VERSION.incrementAndGet()); + DictionaryResponse dictionaryResponse = DictionaryResponse.builder() + .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_author.txt"))) + .build(); + when(dictionaryClient.getDictionaryForType(AUTHOR_CODE, TEST_RULESET_ID)).thenReturn(dictionaryResponse); + DictionaryResponse addressResponse = DictionaryResponse.builder() + .entries(new ArrayList<>(ResourceLoader.load("dictionaries/CBI_address.txt"))) + .build(); + when(dictionaryClient.getDictionaryForType(ADDRESS_CODE, TEST_RULESET_ID)).thenReturn(addressResponse); + DictionaryResponse sponsorResponse = DictionaryResponse.builder() + .entries(Collections.emptyList()) + .build(); + when(dictionaryClient.getDictionaryForType(SPONSOR_CODE, TEST_RULESET_ID)).thenReturn(sponsorResponse); + try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { + Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); + entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null); + assertThat(classifiedDoc.getEntities() + .entrySet() + .stream() + .noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue(); + } + pdfFileResource = new ClassPathResource("files/Compounds/27 A8637C - EU AIR3 - MCP Section 1 - Identity of " + + "the plant protection product.pdf"); + try (PDDocument pdDocument = PDDocument.load(pdfFileResource.getInputStream())) { + Document classifiedDoc = pdfSegmentationService.parseDocument(pdDocument); + entityRedactionService.processDocument(classifiedDoc, TEST_RULESET_ID, null); + assertThat(classifiedDoc.getEntities() + .entrySet() + .stream() + .noneMatch(entry -> entry.getValue().stream().anyMatch(e -> e.getMatchedRule() == 9))).isTrue(); + } + } + + @Test public void testTableRedaction() throws IOException {