Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-6369: Rules Refactor

Browse Source 
*added betweenStrings to entityCreationService
This commit is contained in:
Kilian Schuettler 2023-04-03 14:12:40 +02:00 committed by Kilian Schuettler
parent 6f8242a55e
commit 74a7e43367
7 changed files with 160 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/RegexMatcher.java → redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/CharSequenceSearchUtils.java
View File

@ -11,7 +11,7 @@ import com.iqser.red.service.redaction.v1.server.document.graph.Boundary;
import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock; import com.iqser.red.service.redaction.v1.server.document.graph.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns; import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
public class RegexMatcher { public class CharSequenceSearchUtils {
public static boolean anyMatch(CharSequence charSequence, String regexPattern) { public static boolean anyMatch(CharSequence charSequence, String regexPattern) {
@ -35,7 +35,7 @@ public class RegexMatcher {
} }
public static List<Boundary> findBoundaries(String regexPattern, TextBlock textBlock) { public static List<Boundary> findBoundariesByRegex(String regexPattern, TextBlock textBlock) {
Pattern pattern = Patterns.getCompiledPattern(regexPattern, false); Pattern pattern = Patterns.getCompiledPattern(regexPattern, false);
Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary())); Matcher matcher = pattern.matcher(textBlock.subSequence(textBlock.getBoundary()));
@ -46,4 +46,14 @@ public class RegexMatcher {
return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList(); return boundaries.stream().filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary)).toList();
} }
public static List<Boundary> findBoundariesByString(String searchString, TextBlock textBlock) {
List<Boundary> boundaries = new LinkedList<>();
for (int index = textBlock.indexOf(searchString); index >= 0; index = textBlock.indexOf(searchString, index + 1)) {
boundaries.add(new Boundary(index, index + searchString.length()));
}
return boundaries;
}
} }

48
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/document/services/EntityCreationService.java
View File

@ -1,9 +1,12 @@
package com.iqser.red.service.redaction.v1.server.document.services; package com.iqser.red.service.redaction.v1.server.document.services;
import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.findBoundariesByString;
import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators; import static com.iqser.red.service.redaction.v1.server.redaction.utils.SeparatorUtils.validateBoundaryIsSurroundedBySeparators;
import java.util.Collections; import java.util.Collections;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -30,50 +33,69 @@ public class EntityCreationService {
private final EntityTextEnrichmentService entityEnrichmentService; private final EntityTextEnrichmentService entityEnrichmentService;
public Set<EntityNode> createEntitiesByBetweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) { public Set<EntityNode> betweenStrings(String start, String stop, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock();
List<Boundary> startBoundaries = findBoundariesByString(start, textBlock);
List<Boundary> stopBoundaries = findBoundariesByString(stop, textBlock);
List<Boundary> entityBoundaries = new LinkedList<>();
if (startBoundaries.isEmpty() || stopBoundaries.isEmpty()) {
return Collections.emptySet(); return Collections.emptySet();
} }
for (Boundary startBoundary : startBoundaries) {
Optional<Boundary> optionalFirstStopBoundary = stopBoundaries.stream().filter(stopBoundary -> stopBoundary.end() > startBoundary.start()).findFirst();
if (optionalFirstStopBoundary.isEmpty()) {
break;
}
stopBoundaries.remove(optionalFirstStopBoundary.get());
entityBoundaries.add(new Boundary(startBoundary.end(), optionalFirstStopBoundary.get().start()));
}
return entityBoundaries.stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet());
}
public Set<EntityNode> createEntitiesBySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) { public Set<EntityNode> bySearchImplementation(SearchImplementation searchImplementation, String type, EntityType entityType, SemanticNode node) {
return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary()) return searchImplementation.getBoundaries(node.buildTextBlock(), node.getBoundary())
.stream() .stream()
.filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary)) .filter(boundary -> validateBoundaryIsSurroundedBySeparators(node.buildTextBlock(), boundary))
.map(bounds -> createEntityByBoundary(bounds, type, entityType, node)) .map(bounds -> byBoundary(bounds, type, entityType, node))
.collect(Collectors.toUnmodifiableSet()); .collect(Collectors.toUnmodifiableSet());
} }
public Set<EntityNode> createEntitiesByLineAfterString(String string, String type, EntityType entityType, SemanticNode node) { public Set<EntityNode> lineAfterString(String string, String type, EntityType entityType, SemanticNode node) {
TextBlock textBlock = node.buildTextBlock(); TextBlock textBlock = node.buildTextBlock();
SearchImplementation searchImplementation = new SearchImplementation(List.of(string), true); List<Boundary> boundaries = findBoundariesByString(string, textBlock);
List<Boundary> boundaries = searchImplementation.getBoundaries(textBlock, node.getBoundary());
return boundaries.stream() return boundaries.stream()
.map(boundary -> toLineAfterBoundary(textBlock, boundary)) .map(boundary -> toLineAfterBoundary(textBlock, boundary))
.map(boundary -> createEntityByBoundary(boundary, type, entityType, node)) .filter(boundary -> validateBoundaryIsSurroundedBySeparators(textBlock, boundary))
.map(boundary -> byBoundary(boundary, type, entityType, node))
.collect(Collectors.toUnmodifiableSet()); .collect(Collectors.toUnmodifiableSet());
} }
public Set<EntityNode> createEntitiesByRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) { public Set<EntityNode> byRegex(String regexPattern, String type, EntityType entityType, SemanticNode node) {
List<Boundary> boundaries = RegexMatcher.findBoundaries(regexPattern, node.buildTextBlock()); List<Boundary> boundaries = CharSequenceSearchUtils.findBoundariesByRegex(regexPattern, node.buildTextBlock());
return boundaries.stream().map(boundary -> createEntityByBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet()); return boundaries.stream().map(boundary -> byBoundary(boundary, type, entityType, node)).collect(Collectors.toUnmodifiableSet());
} }
public EntityNode createEntityBySemanticNode(SemanticNode node, String type, EntityType entityType) { public EntityNode bySemanticNode(SemanticNode node, String type, EntityType entityType) {
Boundary boundary = node.buildTextBlock().getBoundary(); Boundary boundary = node.buildTextBlock().getBoundary();
Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1); Boundary nodeBoundaryWithoutTrailingWhitespace = new Boundary(boundary.start(), boundary.end() - 1);
return createEntityByBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node); return byBoundary(nodeBoundaryWithoutTrailingWhitespace, type, entityType, node);
} }
public EntityNode createEntityByBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) { public EntityNode byBoundary(Boundary boundary, String type, EntityType entityType, SemanticNode node) {
EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType); EntityNode entity = EntityNode.initialEntityNode(boundary, type, entityType);
addEntityToGraph(entity, node.getTableOfContents()); addEntityToGraph(entity, node.getTableOfContents());

29
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/redaction/service/entityredaction/EntityRedactionService.java
View File

@ -54,8 +54,9 @@ public class EntityRedactionService {
public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) { public PageEntities findEntities(Dictionary dictionary, List<SectionText> sectionTexts, KieContainer kieContainer, AnalyzeRequest analyzeRequest, NerEntities nerEntities) {
Map<Integer, Set<Image>> imagesPerPage = new HashMap<>(); Map<Integer, Set<Image>> imagesPerPage = new HashMap<>();
long start = System.currentTimeMillis();
FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities); FindEntitiesResult findEntitiesResult = findEntities(sectionTexts, dictionary, kieContainer, analyzeRequest, false, null, imagesPerPage, nerEntities);
System.out.printf("First iteration took %d ms and found %d entities\n", System.currentTimeMillis() - start, findEntitiesResult.getEntities().size());
if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) { if (dictionary.hasLocalEntries() || !findEntitiesResult.getAddedFileAttributes().isEmpty()) {
if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) { if (!findEntitiesResult.getAddedFileAttributes().isEmpty()) {
@ -65,7 +66,7 @@ public class EntityRedactionService {
mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes()); mergedFileAttributes.addAll(findEntitiesResult.getAddedFileAttributes());
analyzeRequest.setFileAttributes(mergedFileAttributes); analyzeRequest.setFileAttributes(mergedFileAttributes);
} }
long start1 = System.currentTimeMillis();
Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary); Map<Integer, Set<Entity>> hintsPerSectionNumber = getHintsPerSection(findEntitiesResult.getEntities(), dictionary);
FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts, FindEntitiesResult foundByLocalEntitiesResult = findEntities(sectionTexts,
dictionary, dictionary,
@ -77,11 +78,12 @@ public class EntityRedactionService {
nerEntities); nerEntities);
EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary); EntitySearchUtils.addEntitiesWithHigherRank(findEntitiesResult.getEntities(), foundByLocalEntitiesResult.getEntities(), dictionary);
EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities()); EntitySearchUtils.removeEntitiesContainedInLarger(findEntitiesResult.getEntities());
System.out.printf("Second iteration took %d ms\n", System.currentTimeMillis() - start1);
} }
Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities()); Map<Integer, List<Entity>> entitiesPerPage = convertToEntitiesPerPage(findEntitiesResult.getEntities());
//EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage); //EntitySearchUtils.removeEntitiesContainedInRedactedLogos(imagesPerPage, entitiesPerPage);
System.out.printf("Total time for extracting entities %d ms\n", System.currentTimeMillis() - start);
return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes()); return new PageEntities(entitiesPerPage, imagesPerPage, findEntitiesResult.getAddedFileAttributes());
} }
@ -95,15 +97,18 @@ public class EntityRedactionService {
Map<Integer, Set<Image>> imagesPerPage, Map<Integer, Set<Image>> imagesPerPage,
NerEntities nerEntities) { NerEntities nerEntities) {
long start = System.currentTimeMillis();
List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections, List<SectionSearchableTextPair> sectionSearchableTextPairs = extractSearchableTextPairs(reanalysisSections,
dictionary, dictionary,
analyzeRequest, analyzeRequest,
local, local,
hintsPerSectionNumber, hintsPerSectionNumber,
nerEntities); nerEntities);
System.out.printf("Total Dict Search and insertion time %d ms\n", System.currentTimeMillis() - start);
Set<FileAttribute> addedFileAttributes = new HashSet<>(); Set<FileAttribute> addedFileAttributes = new HashSet<>();
Set<Entity> entities = new HashSet<>(); Set<Entity> entities = new HashSet<>();
long start1 = System.currentTimeMillis();
sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> { sectionSearchableTextPairs.forEach(sectionSearchableTextPair -> {
if (!addedFileAttributes.isEmpty()) { if (!addedFileAttributes.isEmpty()) {
@ -113,7 +118,9 @@ public class EntityRedactionService {
mergedFileAttributes.addAll(addedFileAttributes); mergedFileAttributes.addAll(addedFileAttributes);
sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes); sectionSearchableTextPair.getSection().setFileAttributes(mergedFileAttributes);
} }
if (sectionSearchableTextPair.getSection() == null) {
return;
}
Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection()); Section analysedSection = droolsExecutionService.executeRules(kieContainer, sectionSearchableTextPair.getSection());
addedFileAttributes.addAll(analysedSection.getAddedFileAttributes()); addedFileAttributes.addAll(analysedSection.getAddedFileAttributes());
@ -137,6 +144,7 @@ public class EntityRedactionService {
} }
}); });
System.out.printf("rules took %d ms in total\n", System.currentTimeMillis() - start1);
return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build(); return FindEntitiesResult.builder().entities(entities).addedFileAttributes(addedFileAttributes).build();
} }
@ -146,14 +154,16 @@ public class EntityRedactionService {
if (cellStarts != null && !cellStarts.isEmpty()) { if (cellStarts != null && !cellStarts.isEmpty()) {
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
searchableText, dictionary, searchableText,
dictionary,
cellStarts, cellStarts,
redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords()); redactionServiceSettings.getNumberOfSurroundingWords());
} else { } else {
SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText, SurroundingWordsService.addSurroundingText(entriesWithoutSurroundingText,
searchableText, dictionary, searchableText,
dictionary,
redactionServiceSettings.getSurroundingWordsOffsetWindow(), redactionServiceSettings.getSurroundingWordsOffsetWindow(),
redactionServiceSettings.getNumberOfSurroundingWords()); redactionServiceSettings.getNumberOfSurroundingWords());
} }
@ -168,7 +178,7 @@ public class EntityRedactionService {
NerEntities nerEntities) { NerEntities nerEntities) {
return reanalysisSections.stream().map(reanalysisSection -> { return reanalysisSections.stream().map(reanalysisSection -> {
long start = System.currentTimeMillis();
Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(), Entities entities = entityFinder.findEntities(reanalysisSection.getSearchableText(),
reanalysisSection.getHeadline(), reanalysisSection.getHeadline(),
reanalysisSection.getSectionNumber(), reanalysisSection.getSectionNumber(),
@ -177,9 +187,7 @@ public class EntityRedactionService {
nerEntities, nerEntities,
reanalysisSection.getCellStarts(), reanalysisSection.getCellStarts(),
analyzeRequest.getManualRedactions()); analyzeRequest.getManualRedactions());
addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities()); addSurroundingText(dictionary, reanalysisSection.getSearchableText(), reanalysisSection.getCellStarts(), entities.getEntities());
if (!local && analyzeRequest.getManualRedactions() != null) { if (!local && analyzeRequest.getManualRedactions() != null) {
var approvedForceRedactions = analyzeRequest.getManualRedactions() var approvedForceRedactions = analyzeRequest.getManualRedactions()
@ -220,7 +228,6 @@ public class EntityRedactionService {
} }
})); }));
} }
log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages()); log.debug("Section {}, Images: {}", reanalysisSection.getSectionNumber(), reanalysisSection.getImages());
return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities); return toSectionSearchableTextPair(dictionary, analyzeRequest, hintsPerSectionNumber, reanalysisSection, entities);

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/DocumentGraphIntegrationTest.java
View File

@ -65,7 +65,7 @@ public class DocumentGraphIntegrationTest extends AbstractTestWithDictionaries {
@SneakyThrows @SneakyThrows
public void testDroolsOnDocumentGraph() { public void testDroolsOnDocumentGraph() {
String filename = "files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06"; String filename = "files/new/crafted document";
prepareStorage(filename + ".pdf"); prepareStorage(filename + ".pdf");
ClassPathResource fileResource = new ClassPathResource(filename + ".pdf"); ClassPathResource fileResource = new ClassPathResource(filename + ".pdf");

14
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentGraphTest.java
View File

@ -56,7 +56,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("Expand to Hint ", entityNode.getTextBefore()); assertEquals("Expand to Hint ", entityNode.getTextBefore());
assertEquals("’s Donut ←", entityNode.getTextAfter()); assertEquals("’s Donut ←", entityNode.getTextAfter());
@ -80,7 +80,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextBefore());
assertEquals(" Purity Hint", entityNode.getTextAfter()); assertEquals(" Purity Hint", entityNode.getTextAfter());
@ -103,7 +103,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("", entityNode.getTextBefore()); assertEquals("", entityNode.getTextBefore());
assertEquals("", entityNode.getTextAfter()); assertEquals("", entityNode.getTextAfter());
@ -189,7 +189,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore()); assertEquals("except Cranberry; Vegetable, ", entityNode.getTextBefore());
assertEquals(", Group 9;", entityNode.getTextAfter()); assertEquals(", Group 9;", entityNode.getTextAfter());
@ -214,7 +214,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("2.6.1 Summary of ", entityNode.getTextBefore()); assertEquals("2.6.1 Summary of ", entityNode.getTextBefore());
assertEquals(" and excretion in", entityNode.getTextAfter()); assertEquals(" and excretion in", entityNode.getTextAfter());
@ -238,7 +238,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore()); assertEquals("2-[(2-(1-hydroxy-ethyl)-6methyl-phenyl-amino]propan-1-ol (", entityNode.getTextBefore());
assertEquals(" of metabolite of", entityNode.getTextAfter()); assertEquals(" of metabolite of", entityNode.getTextAfter());
@ -293,7 +293,7 @@ public class DocumentGraphTest extends AbstractTestWithDictionaries {
assert start != -1; assert start != -1;
Boundary boundary = new Boundary(start, start + searchTerm.length()); Boundary boundary = new Boundary(start, start + searchTerm.length());
EntityNode entityNode = entityCreationService.createEntityByBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph); EntityNode entityNode = entityCreationService.byBoundary(boundary, "CBI_author", EntityType.ENTITY, documentGraph);
PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow(); PageNode pageNode = documentGraph.getPages().stream().filter(page -> page.getNumber() == pageNumber).findFirst().orElseThrow();
assertEquals(entityNode.getValue(), searchTerm); assertEquals(entityNode.getValue(), searchTerm);

82
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/entity_rules.drl
View File

@ -1,7 +1,7 @@
package drools package drools
import static java.lang.String.format; import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields; import static com.iqser.red.service.redaction.v1.server.document.services.EntityFieldSetter.setFields;
@ -78,7 +78,7 @@ rule "5: Add FALSE_POSITIVE Entity for genitive CBI_author"
when when
$entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction) $entity: EntityNode(type == "CBI_author", anyMatch(textAfter, "['’’'ʼˈ´`‘′ʻ’']s"), redaction)
then then
EntityNode entity = entityCreationService.createEntityByBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document); EntityNode entity = entityCreationService.byBoundary($entity.getBoundary(), "CBI_author", EntityType.FALSE_POSITIVE, document);
setFields($entity, 5, "Genitive Author", null, Engine.RULE); setFields($entity, 5, "Genitive Author", null, Engine.RULE);
insert(entity); insert(entity);
end end
@ -90,7 +90,7 @@ rule "6: Create Entity from Author(s) cells in Tables with Author(s) header and
when when
authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText()) authorCell: TableCellNode(!header, (hasHeader("Author(s)") || hasHeader("Author")), hasText())
then then
EntityNode entity = entityCreationService.createEntityBySemanticNode(authorCell, "CBI_author", EntityType.ENTITY); EntityNode entity = entityCreationService.bySemanticNode(authorCell, "CBI_author", EntityType.ENTITY);
setFields(entity, 6, "Header Author(s) found", null, Engine.RULE); setFields(entity, 6, "Header Author(s) found", null, Engine.RULE);
insert(entity); insert(entity);
dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true); dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), true);
@ -102,7 +102,7 @@ rule "7: Add CBI_author with \"et al.\" Regex"
when when
$section: SectionNode(containsString("et al.")) $section: SectionNode(containsString("et al."))
then then
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section); Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, $section);
setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE); setFields(entities, 7, "Found by \"et al.\" regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity)); entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false)); entities.forEach(entity -> dictionary.addLocalDictionaryEntry("CBI_author", entity.getValue(), false));
@ -112,9 +112,9 @@ rule "8: Add recommendation for Addresses in Test Organism sections"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species") && containsString("Source")) $section: SectionNode(containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
then then
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section); Set<EntityNode> entities = entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE); setFields(entities, 8, "Line after \"Source\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity)); entities.forEach(entity -> insert(entity));
end end
@ -126,7 +126,7 @@ rule "9: Add recommendation for Addresses in Test Animals sections"
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes") FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("Species:") && containsString("Source:")) $section: SectionNode(containsString("Species:") && containsString("Source:"))
then then
Set<EntityNode> entities = entityCreationService.createEntitiesByLineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section); Set<EntityNode> entities = entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section);
setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE); setFields(entities, 9, "Line after \"Source:\" in Test Organism Section", null, Engine.RULE);
entities.forEach(entity -> insert(entity)); entities.forEach(entity -> insert(entity));
end end
@ -163,9 +163,75 @@ rule "11: Redacted PII Personal Identification Information (Vertebrate study)"
rule "12: Redact Emails by RegEx (Non vertebrate study)" rule "12: Redact Emails by RegEx (Non vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode(containsString("@")) $section: SectionNode(containsString("@"))
then then
Set<EntityNode> entities = entityCreationService.createEntitiesByRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section); Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 12, "Found by email regex", null, Engine.RULE); setFields(entities, 12, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity)); entities.forEach(entity -> insert(entity));
end end
rule "13: Redact Emails by RegEx (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode(containsString("@"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 13, "Found by email regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end
rule "14: Redact contact information (Non vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() != "yes")
$section: SectionNode()
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
setFields(entities, 14, "Found by rule based keywords", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "15: Redact contact information (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study" , value.toLowerCase() == "yes")
$section: SectionNode()
then
Set<EntityNode> entities = new HashSet<>();
entities.addAll(entityCreationService.lineAfterString("Contact point:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("Alternative contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.lineAfterString("European contact:", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section));
entities.addAll(entityCreationService.betweenStrings("Contact:", "Tel.:", "PII", EntityType.ENTITY, $section));
setFields(entities, 15, "Found by contacts keywords", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
entities.forEach(entity -> dictionary.addLocalDictionaryEntry("PII", entity.getValue(), false));
end
rule "16: Redact Phone and Fax by RegEx"
when
$section: SectionNode(containsString("Contact") ||
containsString("Telephone") ||
containsString("Phone") ||
containsString("Fax") ||
containsString("Tel") ||
containsString("Ter") ||
containsString("Mobile") ||
containsString("Fel") ||
containsString("Fer"))
then
Set<EntityNode> entities = entityCreationService.byRegex("\\b(contact|telephone|phone|fax|tel|ter|mobile|fel|fer)[a-zA-Z\\s]{0,10}[:.\\s]{0,3}([\\+\\d\\(][\\s\\d\\(\\)\\-\\/\\.]{4,100}\\d)\\b", "PII", EntityType.ENTITY, $section);
setFields(entities, 16, "Found by Phone and Fax regex", null, Engine.RULE);
entities.forEach(entity -> insert(entity));
end

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/merge_entity_rules.drl
View File

@ -1,7 +1,7 @@
package drools package drools
import static java.lang.String.format; import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.document.services.RegexMatcher.anyMatch; import static com.iqser.red.service.redaction.v1.server.document.services.CharSequenceSearchUtils.anyMatch;
import java.util.List; import java.util.List;
@ -34,8 +34,8 @@ rule "merge intersecting Entities of same type"
$second.removeFromGraph(); $second.removeFromGraph();
EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document); EntityNode mergedEntity = entityCreationService.createMergedEntity(List.of($first, $second), $type, $entityType, document);
insert(mergedEntity); insert(mergedEntity);
delete($first); retract($first);
delete($second); retract($second);
end end
rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE" rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
@ -46,7 +46,7 @@ rule "remove Entity of type ENTITY when contained by FALSE_POSITIVE"
entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger) entity: EntityNode(containedBy($falsePositive), type == $type, entityType == EntityType.ENTITY, !resized, !skipRemoveEntitiesContainedInLarger)
then then
entity.removeFromGraph(); entity.removeFromGraph();
delete(entity); retract(entity);
end end
rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
@ -57,7 +57,7 @@ rule "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATIO
$recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) $recommendation: EntityNode(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then then
$recommendation.removeFromGraph(); $recommendation.removeFromGraph();
delete($recommendation); retract($recommendation);
end end
rule "remove Entity of type RECOMMENDATION when contained by ENTITY" rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
@ -68,7 +68,7 @@ rule "remove Entity of type RECOMMENDATION when contained by ENTITY"
$recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger) $recommendation: EntityNode(containedBy($entity), type == $type, entityType == EntityType.RECOMMENDATION, !resized, !skipRemoveEntitiesContainedInLarger)
then then
$recommendation.removeFromGraph(); $recommendation.removeFromGraph();
delete($recommendation); retract($recommendation);
end end
rule "remove Entity of lower rank, when equal boundaries and entityType" rule "remove Entity of lower rank, when equal boundaries and entityType"
@ -79,7 +79,7 @@ rule "remove Entity of lower rank, when equal boundaries and entityType"
$lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type)) $lowerRank: EntityNode($boundary == boundary, type != $type, entityType == $entityType, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type))
then then
$lowerRank.removeFromGraph(); $lowerRank.removeFromGraph();
delete($lowerRank); retract($lowerRank);
end end
rule "run local dictionary search" rule "run local dictionary search"
@ -89,7 +89,7 @@ rule "run local dictionary search"
when when
DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels() DictionaryModel(!localEntries.isEmpty(), $type: type, $searchImplementation: localSearch) from dictionary.getDictionaryModels()
then then
Set<EntityNode> entityNodes = entityCreationService.createEntitiesBySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document); Set<EntityNode> entityNodes = entityCreationService.bySearchImplementation($searchImplementation, $type, EntityType.RECOMMENDATION, document);
System.out.printf("local dictionary search found %d entities\n", entityNodes.size()); System.out.printf("local dictionary search found %d entities\n", entityNodes.size());
entityNodes.forEach(entityNode -> insert(entityNode)); entityNodes.forEach(entityNode -> insert(entityNode));
end end