diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java similarity index 77% rename from redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java rename to redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java index 8b20d1fd..391566ed 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java @@ -11,7 +11,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.Boundary; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; -public class RegexMatcher { +public class CharSequenceSearchUtils { public static boolean anyMatch(CharSequence charSequence, String regexPattern) { @@ -35,7 +35,7 @@ public class RegexMatcher { } - public static List findBoundaries(String regexPattern, TextBlock textBlock) { + public static List findBoundariesByRegex(String regexPattern, TextBlock textBlock) { Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary())); @@ -46,4 +46,14 @@ public class RegexMatcher { return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList(); } + + public static List findBoundariesByString(String searchString, TextBlock textBlock) { + + List boundaries = new LinkedList<>(); + for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) { + boundaries.add(new Boundary(index, index + searchString.length())); + } + return boundaries; + } + } diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java index 76ca8656..b38333d1 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java @@ -1,9 +1,12 @@ package com.iqser.red.service.redaction.v1.server.document.services; +import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; import java.util.Collections; +import java.util.LinkedList; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -30,50 +33,69 @@ public class EntityCreationService { private final EntityTextEnrichmentService entityEnrichmentService; - public Set createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { + public Set betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { - return Collections.emptySet(); + TextBlock textBlock = node.buildTextBlock(); + List startBoundaries = findBoundariesByString(start, textBlock); + List stopBoundaries = findBoundariesByString(stop, textBlock); + + List entityBoundaries = new LinkedList<>(); + if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) { + return Collections.emptySet(); + } + for (Boundary startBoundary : startBoundaries) { + Optional optionalFirstStopBoundary = stopBoundaries.stream().filter(stopBoundary -> stopBoundary.end() > startBoundary.start()).findFirst(); + if (optionalFirstStopBoundary.isEmpty()) { + break; + } + stopBoundaries.remove(optionalFirstStopBoundary.get()); + entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start())); + } + return entityBoundaries.stream() + .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) + .map(boundary -> byBoundary(boundary, type, entityType, node)) + .collect(Collectors.toUnmodifiableSet()); } - public Set createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { + public Set bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) .stream() .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary)) - .map(bounds -> createEntityByBoundary(bounds, type, entityType, node)) + .map(bounds -> byBoundary(bounds, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } - public Set createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) { + public Set lineAfterString(String string, String type, EntityType entityType, SemanticNode node) { TextBlock textBlock = node.buildTextBlock(); - SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true); - List boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary()); + List boundaries = findBoundariesByString(string, textBlock); return boundaries.stream() .map(boundary -> toLineAfterBoundary(textBlock, boundary)) - .map(boundary -> createEntityByBoundary(boundary, type, entityType, node)) + .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)) + .map(boundary -> byBoundary(boundary, type, entityType, node)) .collect(Collectors.toUnmodifiableSet()); } - public Set createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { + public Set byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { - List boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock()); - return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet()); + List boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock()); + return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet()); } - public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) { + public EntityNode bySemanticNode(SemanticNode node, String type, EntityType entityType) { Boundary boundary = node.buildTextBlock().getBoundary(); Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); - return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node); + return byBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node); } - public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { + public EntityNode byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); addEntityToGraph(entity, node.getTableOfContents()); diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java index c0c9cd4a..f286f556 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java @@ -54,8 +54,9 @@ public class EntityRedactionService { public PageEntities findEntities(Dictionary dictionary, List sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { Map> imagesPerPage = new HashMap<>(); + long start = System.currentTimeMillis(); FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities); - + System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size()); if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) { if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) { @@ -65,7 +66,7 @@ public class EntityRedactionService { mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes()); analyzeRequest.setFileAttributes(mergedFileAttributes); } - + long start1 = System.currentTimeMillis(); Map> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary); FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, dictionary, @@ -77,11 +78,12 @@ public class EntityRedactionService { nerEntities); EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary); EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities()); + System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1); } Map> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities()); //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage); - + System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start); return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes()); } @@ -95,15 +97,18 @@ public class EntityRedactionService { Map> imagesPerPage, NerEntities nerEntities) { + long start = System.currentTimeMillis(); List sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections, dictionary, analyzeRequest, local, hintsPerSectionNumber, nerEntities); - + System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start); Set addedFileAttributes = new HashSet<>(); Set entities = new HashSet<>(); + + long start1 = System.currentTimeMillis(); sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { if (!addedFileAttributes.isEmpty()) { @@ -113,7 +118,9 @@ public class EntityRedactionService { mergedFileAttributes.addAll(addedFileAttributes); sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes); } - + if (sectionSearchableTextPair.getSection() == null) { + return; + } Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); addedFileAttributes.addAll(analysedSection.getAddedFileAttributes()); @@ -137,23 +144,26 @@ public class EntityRedactionService { } }); + System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1); return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build(); } - private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List cellStarts, Set entriesWithoutSurroundingText) { + private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List cellStarts, Set entriesWithoutSurroundingText) { if (cellStarts != null && !cellStarts.isEmpty()) { SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - searchableText, dictionary, + searchableText, + dictionary, cellStarts, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); } else { SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, - searchableText, dictionary, + searchableText, + dictionary, redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getNumberOfSurroundingWords()); } @@ -168,7 +178,7 @@ public class EntityRedactionService { NerEntities nerEntities) { return reanalysisSections.stream().map(reanalysisSection -> { - + long start = System.currentTimeMillis(); Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(), reanalysisSection.getHeadline(), reanalysisSection.getSectionNumber(), @@ -177,9 +187,7 @@ public class EntityRedactionService { nerEntities, reanalysisSection.getCellStarts(), analyzeRequest.getManualRedactions()); - addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities()); - if (!local && analyzeRequest.getManualRedactions() != null) { var approvedForceRedactions = analyzeRequest.getManualRedactions() @@ -220,7 +228,6 @@ public class EntityRedactionService { } })); } - log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages()); return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java index 876cdf1c..a36b8333 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java @@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries { @SneakyThrows public void testDroolsOnDocumentGraph() { - String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; + String filename = "files/new/crafted document"; prepareStorage(filename + ".pdf"); ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java index 4186b6d6..d39f93bb 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java @@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("’s Donut ←", entityNode.getTextAfter()); @@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals(" Purity Hint", entityNode.getTextAfter()); @@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextAfter()); @@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals(", Group 9;", entityNode.getTextAfter()); @@ -214,7 +214,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("2.6.1 Summary of ", entityNode.getTextBefore()); assertEquals(" and excretion in", entityNode.getTextAfter()); @@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals(" of metabolite of", entityNode.getTextAfter()); @@ -293,7 +293,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries { assert start != -1; Boundary boundary = new Boundary(start, start + searchTerm.length()); - EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); + EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); assertEquals(entityNode.getValue(), searchTerm); diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl index 879cebbb..52b30e68 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl @@ -1,7 +1,7 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields; @@ -78,7 +78,7 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author" when $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) then - EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); + EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); setFields($entity, 5, "Genitive Author", null, Engine.RULE); insert(entity); end @@ -90,7 +90,7 @@ rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and when authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText()) then - EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); + EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); setFields(entity, 6, "Header Author(s) found", null, Engine.RULE); insert(entity); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true); @@ -102,7 +102,7 @@ rule "7: Add CBI_author with \"et al.\" Regex" when $section: SectionNode(containsString("et al.")) then - Set entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); + Set entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE); entities.forEach(entity -> insert(entity)); entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false)); @@ -112,9 +112,9 @@ rule "8: Add recommendation for Addresses in Test Organism sections" when FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") - $section: SectionNode(containsString("Species") && containsString("Source")) + $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:")) then - Set entities = entityCreationService.createEntitiesByLineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); + Set entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE); entities.forEach(entity -> insert(entity)); end @@ -126,7 +126,7 @@ rule "9: Add recommendation for Addresses in Test Animals sections" FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") $section: SectionNode(containsString("Species:") && containsString("Source:")) then - Set entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); + Set entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE); entities.forEach(entity -> insert(entity)); end @@ -163,9 +163,75 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)" rule "12: Redact Emails by RegEx (Non vertebrate study)" when - $section: SectionNode(containsString("@")) + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode(containsString("@")) then - Set entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); - setFields(entities, 12, "Found by email regex", null, Engine.RULE); - entities.forEach(entity -> insert(entity)); + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 12, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + end + +rule "13: Redact Emails by RegEx (Vertebrate study)" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode(containsString("@")) + then + Set entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 13, "Found by email regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + end + +rule "14: Redact contact information (Non vertebrate study)" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes") + $section: SectionNode() + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section)); + setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "15: Redact contact information (Vertebrate study)" + + when + FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") + $section: SectionNode() + then + Set entities = new HashSet<>(); + entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section)); + entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section)); + setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); + entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false)); + end + +rule "16: Redact Phone and Fax by RegEx" + + when + $section: SectionNode(containsString("Contact") || + containsString("Telephone") || + containsString("Phone") || + containsString("Fax") || + containsString("Tel") || + containsString("Ter") || + containsString("Mobile") || + containsString("Fel") || + containsString("Fer")) + then + Set entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section); + setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE); + entities.forEach(entity -> insert(entity)); end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl index f28c0fe1..39b8a3f9 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl @@ -1,7 +1,7 @@ package drools import static java.lang.String.format; -import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; +import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch; import java.util.List; @@ -34,8 +34,8 @@ rule "merge intersecting Entities of same type" $second.removeFromGraph(); EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document); insert(mergedEntity); - delete($first); - delete($second); + retract($first); + retract($second); end rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" @@ -46,7 +46,7 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) then entity.removeFromGraph(); - delete(entity); + retract(entity); end rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" @@ -57,7 +57,7 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $recommendation.removeFromGraph(); - delete($recommendation); + retract($recommendation); end rule "remove Entity of type RECOMMENDATION when contained by ENTITY" @@ -68,7 +68,7 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY" $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) then $recommendation.removeFromGraph(); - delete($recommendation); + retract($recommendation); end rule "remove Entity of lower rank, when equal boundaries and entityType" @@ -79,7 +79,7 @@ rule "remove Entity of lower rank, when equal boundaries and entityType" $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type)) then $lowerRank.removeFromGraph(); - delete($lowerRank); + retract($lowerRank); end rule "run local dictionary search" @@ -89,7 +89,7 @@ rule "run local dictionary search" when DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() then - Set entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); + Set entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); System.out.printf("local dictionary search found %d entities\n", entityNodes.size()); entityNodes.forEach(entityNode -> insert(entityNode)); end