RED-6369: Rules Refactor
*added betweenStrings to entityCreationService
This commit is contained in:
Kilian Schuettler
Kilian Schuettler
parent
b9d93e44bd
commit
482368b651
@ -11,7 +11,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
|
||||
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
|
||||
public class RegexMatcher {
|
||||
public class CharSequenceSearchUtils {
|
||||
|
||||
public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
|
||||
|
||||
@ -35,7 +35,7 @@ public class RegexMatcher {
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundaries(String regexPattern, TextBlock textBlock) {
|
||||
public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
|
||||
|
||||
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
|
||||
Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
|
||||
@ -46,4 +46,14 @@ public class RegexMatcher {
|
||||
return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
|
||||
}
|
||||
|
||||
|
||||
public static List<Boundary> findBoundariesByString(String searchString, TextBlock textBlock) {
|
||||
|
||||
List<Boundary> boundaries = new LinkedList<>();
|
||||
for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) {
|
||||
boundaries.add(new Boundary(index, index + searchString.length()));
|
||||
}
|
||||
return boundaries;
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,9 +1,12 @@
|
||||
package com.iqser.red.service.redaction.v1.server.document.services;
|
||||
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
|
||||
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@ -30,50 +33,69 @@ public class EntityCreationService {
|
||||
private final EntityTextEnrichmentService entityEnrichmentService;
|
||||
|
||||
|
||||
public Set<EntityNode> createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return Collections.emptySet();
|
||||
TextBlock textBlock = node.buildTextBlock();
|
||||
List<Boundary> startBoundaries = findBoundariesByString(start, textBlock);
|
||||
List<Boundary> stopBoundaries = findBoundariesByString(stop, textBlock);
|
||||
|
||||
List<Boundary> entityBoundaries = new LinkedList<>();
|
||||
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
|
||||
return Collections.emptySet();
|
||||
}
|
||||
for (Boundary startBoundary : startBoundaries) {
|
||||
Optional<Boundary> optionalFirstStopBoundary = stopBoundaries.stream().filter(stopBoundary -> stopBoundary.end() > startBoundary.start()).findFirst();
|
||||
if (optionalFirstStopBoundary.isEmpty()) {
|
||||
break;
|
||||
}
|
||||
stopBoundaries.remove(optionalFirstStopBoundary.get());
|
||||
entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
|
||||
}
|
||||
return entityBoundaries.stream()
|
||||
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<EntityNode> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
|
||||
.stream()
|
||||
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
|
||||
.map(bounds -> createEntityByBoundary(bounds, type, entityType, node))
|
||||
.map(bounds -> byBoundary(bounds, type, entityType, node))
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
TextBlock textBlock = node.buildTextBlock();
|
||||
SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true);
|
||||
List<Boundary> boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary());
|
||||
List<Boundary> boundaries = findBoundariesByString(string, textBlock);
|
||||
return boundaries.stream()
|
||||
.map(boundary -> toLineAfterBoundary(textBlock, boundary))
|
||||
.map(boundary -> createEntityByBoundary(boundary, type, entityType, node))
|
||||
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
|
||||
.map(boundary -> byBoundary(boundary, type, entityType, node))
|
||||
.collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
|
||||
public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
public Set<EntityNode> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock());
|
||||
return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
|
||||
List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
|
||||
return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
|
||||
}
|
||||
|
||||
|
||||
public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) {
|
||||
public EntityNode bySemanticNode(SemanticNode node, String type, EntityType entityType) {
|
||||
|
||||
Boundary boundary = node.buildTextBlock().getBoundary();
|
||||
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
|
||||
return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
|
||||
return byBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
|
||||
}
|
||||
|
||||
|
||||
public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
public EntityNode byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
|
||||
|
||||
EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
|
||||
addEntityToGraph(entity, node.getTableOfContents());
|
||||
|
||||
@ -54,8 +54,9 @@ public class EntityRedactionService {
|
||||
public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
|
||||
|
||||
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
|
||||
long start = System.currentTimeMillis();
|
||||
FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
|
||||
|
||||
System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size());
|
||||
if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
||||
|
||||
if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
|
||||
@ -65,7 +66,7 @@ public class EntityRedactionService {
|
||||
mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes());
|
||||
analyzeRequest.setFileAttributes(mergedFileAttributes);
|
||||
}
|
||||
|
||||
long start1 = System.currentTimeMillis();
|
||||
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
|
||||
FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
|
||||
dictionary,
|
||||
@ -77,11 +78,12 @@ public class EntityRedactionService {
|
||||
nerEntities);
|
||||
EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
|
||||
EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
|
||||
System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1);
|
||||
}
|
||||
|
||||
Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
|
||||
//EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
|
||||
|
||||
System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start);
|
||||
return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
|
||||
}
|
||||
|
||||
@ -95,15 +97,18 @@ public class EntityRedactionService {
|
||||
Map<Integer, Set<Image>> imagesPerPage,
|
||||
NerEntities nerEntities) {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
|
||||
dictionary,
|
||||
analyzeRequest,
|
||||
local,
|
||||
hintsPerSectionNumber,
|
||||
nerEntities);
|
||||
|
||||
System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start);
|
||||
Set<FileAttribute> addedFileAttributes = new HashSet<>();
|
||||
Set<Entity> entities = new HashSet<>();
|
||||
|
||||
long start1 = System.currentTimeMillis();
|
||||
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
|
||||
|
||||
if (!addedFileAttributes.isEmpty()) {
|
||||
@ -113,7 +118,9 @@ public class EntityRedactionService {
|
||||
mergedFileAttributes.addAll(addedFileAttributes);
|
||||
sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
|
||||
}
|
||||
|
||||
if (sectionSearchableTextPair.getSection() == null) {
|
||||
return;
|
||||
}
|
||||
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
|
||||
|
||||
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
|
||||
@ -137,23 +144,26 @@ public class EntityRedactionService {
|
||||
}
|
||||
|
||||
});
|
||||
System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1);
|
||||
|
||||
return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
|
||||
}
|
||||
|
||||
|
||||
private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List<Integer> cellStarts, Set<Entity> entriesWithoutSurroundingText) {
|
||||
private void addSurroundingText(Dictionary dictionary, SearchableText searchableText, List<Integer> cellStarts, Set<Entity> entriesWithoutSurroundingText) {
|
||||
|
||||
if (cellStarts != null && !cellStarts.isEmpty()) {
|
||||
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
|
||||
searchableText, dictionary,
|
||||
searchableText,
|
||||
dictionary,
|
||||
cellStarts,
|
||||
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
|
||||
redactionServiceSettings.getNumberOfSurroundingWords());
|
||||
|
||||
} else {
|
||||
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
|
||||
searchableText, dictionary,
|
||||
searchableText,
|
||||
dictionary,
|
||||
redactionServiceSettings.getSurroundingWordsOffsetWindow(),
|
||||
redactionServiceSettings.getNumberOfSurroundingWords());
|
||||
}
|
||||
@ -168,7 +178,7 @@ public class EntityRedactionService {
|
||||
NerEntities nerEntities) {
|
||||
|
||||
return reanalysisSections.stream().map(reanalysisSection -> {
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
|
||||
reanalysisSection.getHeadline(),
|
||||
reanalysisSection.getSectionNumber(),
|
||||
@ -177,9 +187,7 @@ public class EntityRedactionService {
|
||||
nerEntities,
|
||||
reanalysisSection.getCellStarts(),
|
||||
analyzeRequest.getManualRedactions());
|
||||
|
||||
addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities());
|
||||
|
||||
if (!local && analyzeRequest.getManualRedactions() != null) {
|
||||
|
||||
var approvedForceRedactions = analyzeRequest.getManualRedactions()
|
||||
@ -220,7 +228,6 @@ public class EntityRedactionService {
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
|
||||
|
||||
return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);
|
||||
|
||||
@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
|
||||
@SneakyThrows
|
||||
public void testDroolsOnDocumentGraph() {
|
||||
|
||||
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06";
|
||||
String filename = "files/new/crafted document";
|
||||
|
||||
prepareStorage(filename + ".pdf");
|
||||
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");
|
||||
|
||||
@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("Expand to Hint ", entityNode.getTextBefore());
|
||||
assertEquals("’s Donut ←", entityNode.getTextAfter());
|
||||
@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("", entityNode.getTextBefore());
|
||||
assertEquals(" Purity Hint", entityNode.getTextAfter());
|
||||
@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("", entityNode.getTextBefore());
|
||||
assertEquals("", entityNode.getTextAfter());
|
||||
@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
|
||||
assertEquals(", Group 9;", entityNode.getTextAfter());
|
||||
@ -214,7 +214,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("2.6.1 Summary of ", entityNode.getTextBefore());
|
||||
assertEquals(" and excretion in", entityNode.getTextAfter());
|
||||
@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
|
||||
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
|
||||
assertEquals(" of metabolite of", entityNode.getTextAfter());
|
||||
@ -293,7 +293,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
|
||||
assert start != -1;
|
||||
|
||||
Boundary boundary = new Boundary(start, start + searchTerm.length());
|
||||
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
|
||||
PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
|
||||
|
||||
assertEquals(entityNode.getValue(), searchTerm);
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package drools
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
|
||||
|
||||
|
||||
@ -78,7 +78,7 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
|
||||
when
|
||||
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
|
||||
then
|
||||
EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
|
||||
EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
|
||||
setFields($entity, 5, "Genitive Author", null, Engine.RULE);
|
||||
insert(entity);
|
||||
end
|
||||
@ -90,7 +90,7 @@ rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and
|
||||
when
|
||||
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
|
||||
then
|
||||
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
|
||||
EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
|
||||
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
|
||||
insert(entity);
|
||||
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
|
||||
@ -102,7 +102,7 @@ rule "7: Add CBI_author with \"et al.\" Regex"
|
||||
when
|
||||
$section: SectionNode(containsString("et al."))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
|
||||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
|
||||
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
|
||||
@ -112,9 +112,9 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode(containsString("Species") && containsString("Source"))
|
||||
$section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||||
Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||||
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
@ -126,7 +126,7 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode(containsString("Species:") && containsString("Source:"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||||
Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
|
||||
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
@ -163,9 +163,75 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
|
||||
rule "12: Redact Emails by RegEx (Non vertebrate study)"
|
||||
|
||||
when
|
||||
$section: SectionNode(containsString("@"))
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||||
$section: SectionNode(containsString("@"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
|
||||
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
|
||||
setFields(entities, 12, "Found by email regex", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
rule "13: Redact Emails by RegEx (Vertebrate study)"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode(containsString("@"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
|
||||
setFields(entities, 13, "Found by email regex", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
rule "14: Redact contact information (Non vertebrate study)"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
|
||||
$section: SectionNode()
|
||||
then
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
|
||||
setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||||
end
|
||||
|
||||
rule "15: Redact contact information (Vertebrate study)"
|
||||
|
||||
when
|
||||
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
|
||||
$section: SectionNode()
|
||||
then
|
||||
Set<EntityNode> entities = new HashSet<>();
|
||||
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
|
||||
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
|
||||
setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
|
||||
end
|
||||
|
||||
rule "16: Redact Phone and Fax by RegEx"
|
||||
|
||||
when
|
||||
$section: SectionNode(containsString("Contact") ||
|
||||
containsString("Telephone") ||
|
||||
containsString("Phone") ||
|
||||
containsString("Fax") ||
|
||||
containsString("Tel") ||
|
||||
containsString("Ter") ||
|
||||
containsString("Mobile") ||
|
||||
containsString("Fel") ||
|
||||
containsString("Fer"))
|
||||
then
|
||||
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
|
||||
setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE);
|
||||
entities.forEach(entity -> insert(entity));
|
||||
end
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package drools
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch;
|
||||
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
@ -34,8 +34,8 @@ rule "merge intersecting Entities of same type"
|
||||
$second.removeFromGraph();
|
||||
EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
|
||||
insert(mergedEntity);
|
||||
delete($first);
|
||||
delete($second);
|
||||
retract($first);
|
||||
retract($second);
|
||||
end
|
||||
|
||||
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||
@ -46,7 +46,7 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
||||
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
|
||||
then
|
||||
entity.removeFromGraph();
|
||||
delete(entity);
|
||||
retract(entity);
|
||||
end
|
||||
|
||||
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
||||
@ -57,7 +57,7 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO
|
||||
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||||
then
|
||||
$recommendation.removeFromGraph();
|
||||
delete($recommendation);
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||||
@ -68,7 +68,7 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
|
||||
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
|
||||
then
|
||||
$recommendation.removeFromGraph();
|
||||
delete($recommendation);
|
||||
retract($recommendation);
|
||||
end
|
||||
|
||||
rule "remove Entity of lower rank, when equal boundaries and entityType"
|
||||
@ -79,7 +79,7 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
|
||||
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
|
||||
then
|
||||
$lowerRank.removeFromGraph();
|
||||
delete($lowerRank);
|
||||
retract($lowerRank);
|
||||
end
|
||||
|
||||
rule "run local dictionary search"
|
||||
@ -89,7 +89,7 @@ rule "run local dictionary search"
|
||||
when
|
||||
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
|
||||
then
|
||||
Set<EntityNode> entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
|
||||
Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
|
||||
System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
|
||||
entityNodes.forEach(entityNode -> insert(entityNode));
|
||||
end
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user