Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Resolve RED-7679 "Bbb3"

Browse Source
This commit is contained in:
Ali Oezyetimoglu 2023-11-14 10:51:28 +01:00
parent 3a685e5671
commit 71789e12fa
8 changed files with 600 additions and 627 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-api-v1/build.gradle.kts
View File

@ -7,7 +7,7 @@ description = "redaction-service-api-v1"
dependencies {
implementation("org.springframework:spring-web:6.0.12")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0")
implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0")
}
publishing {

56
redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java
View File

@ -19,6 +19,7 @@ import org.kie.api.runtime.KieSession;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
@ -102,7 +103,11 @@ public class ComponentCreationService {
referencedEntities.addAll(references);
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription)
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name)
.value(value)
.valueDescription(valueDescription)
.references(new LinkedList<>(references))
.build());
}
@ -293,6 +298,30 @@ public class ComponentCreationService {
}
/**
* Counts the distinct rows where any of the provided entities occur and creates a component for each distinct table with its respective row count.
*
* @param ruleIdentifier the identifier of the rule
* @param name the name of the record
* @param entities the collection of entities to count row values from
*/
public void rowValueCount(String ruleIdentifier, String name, Collection<Entity> entities) {
entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> {
if (optionalTable.isEmpty()) {
return;
}
long count = groupedEntities.stream()
.collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1)))
.size();
create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities);
});
}
/**
* Creates a component for each sentence in the collection of entities.
*
@ -354,7 +383,9 @@ public class ComponentCreationService {
*/
public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection<Entity> entities) {
entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first())
entities.stream()
.filter(entity -> !referencedEntities.contains(entity))
.sorted(EntityComparators.first())
.forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity)));
}
@ -429,7 +460,8 @@ public class ComponentCreationService {
.sorted(Comparator.comparingInt(Map.Entry::getKey))
.map(Map.Entry::getValue)
.forEach(entitiesInSameRow -> create(ruleIdentifier,
name, entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
name,
entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")),
valueDescription,
entitiesInSameRow));
});
@ -450,6 +482,20 @@ public class ComponentCreationService {
}
private Optional<TableCell> getFirstTableCell(Entity entity) {
SemanticNode node = entity.getContainingNode();
while (!(node instanceof TableCell)) {
if (!node.hasParent()) {
return Optional.empty();
}
node = node.getParent();
}
return Optional.of((TableCell) node);
}
/**
* Creates a new component with the given rule identifier, name, value, and value description.
* If the component is part of a table, it also takes a list of entities that belong to the same table row.
@ -474,7 +520,9 @@ public class ComponentCreationService {
*/
public void create(String ruleIdentifier, String name, String value) {
kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name)
kieSession.insert(Component.builder()
.matchedRule(RuleIdentifier.fromString(ruleIdentifier))
.name(name)
.value(value)
.valueDescription("")
.references(Collections.emptyList())

15
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java
View File

@ -43,33 +43,30 @@ import com.knecon.fforesight.tenantcommons.TenantContext;
@Import(AnalysisTest.RedactionIntegrationTestConfiguration.class)
public class AnalysisTest extends AbstractRedactionIntegrationTest {
private static final String RULES = loadFromClassPath("drools/table_demo.drl");
private static final String COMPONENT_RULES = loadFromClassPath("drools/table_demo_components.drl");
private static final String RULES = loadFromClassPath("drools/test_rules.drl");
private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl");
@Test
@Disabled
public void analyzeTableDemoFile() {
public void analyzeTableDemoFile() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/TableDemo/Table_examples.pdf");
AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf");
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request);
System.out.println("Finished structure analysis");
AnalyzeResult result = analyzeService.analyze(request);
System.out.println("Finished analysis");
// var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID);
// var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID);
AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build());
String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf";
try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) {
fileOutputStream.write(annotateResponse.getDocument());
} catch (IOException e) {
throw new RuntimeException(e);
}
}

91
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -53,7 +53,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
@ -65,7 +65,23 @@ global Dictionary dictionary
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//---------------------------------------------------------------------------
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end
// --------------------------------------- Your rules below this line --------------------------------------------------
rule "TAB.0.0: Study Type File Attribute"
when
@ -106,7 +122,7 @@ rule "TAB.1.0: Full Table extraction (Guideline Deviation)"
$table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
$tableCell: TableCell(!header) from $table.streamTableCells().toList()
then
entityCreationService.bySemanticNode($tableCell, "guideline_deviation", EntityType.ENTITY)
entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted"));
end
@ -139,28 +155,26 @@ rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Combined Columns"))
$table: Table(hasHeader("Sex"), hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
$maleCells: TableCell($row: row, containsAnyWordIgnoreCase("Male")) from $table.streamTableCellsWithHeader("Sex").toList()
$dosageCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList()
$table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
then
entityCreationService.bySemanticNode($maleCells, "combined_male_dosage", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male"));
entityCreationService.bySemanticNode($dosageCells, "combined_male_dosage", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male"));
$table.streamTableCellsWithHeader("Dosage (mg/kg bw)")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found."));
end
rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality"
when
FileAttribute(label == "OECD Number", valueEqualsAnyOf("425"))
$section: Section(getHeadline().containsString("Combined Columns"))
$table: Table(hasHeader("Sex"), hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
$femaleCells: TableCell($row: row, containsAnyWordIgnoreCase("Female")) from $table.streamTableCellsWithHeader("Sex").toList()
$mortalityCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Mortality").toList()
$table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList()
then
entityCreationService.bySemanticNode($femaleCells, "combined_female_mortality", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female"));
entityCreationService.bySemanticNode($mortalityCells, "combined_female_mortality", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female"));
$table.streamTableCellsWithHeader("Mortality")
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY))
.filter(Optional::isPresent)
.map(Optional::get)
.forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found."));
end
rule "TAB.5.0: Targeted cell extraction"
@ -194,7 +208,7 @@ rule "TAB.7.0: Indicator (Species)"
$section: Section(getHeadline().containsString("Entity-Based"))
$table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList()
TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList()
TableCell(hasEntitiesOfType("vertebrate"), $row: row) from $table.streamTableCells().toList()
TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList()
$cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList()
then
entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY)
@ -407,6 +421,31 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
end
// Rule unit: X.6
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
// Rule unit: X.7
rule "X.7.0: remove all images"
salience 512
@ -430,19 +469,3 @@ rule "FA.1.0: Remove duplicate FileAttributes"
retract($duplicate);
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end

134
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_OLD.drl
View File

@ -1,134 +0,0 @@
package drools
import com.iqser.red.service.redaction.v1.server.redaction.model.Section
global Section section
// --------------------------------------- Your rules below this line--------------------------------------------------
rule "0a: Study Type File Attribute"
when
Section(
!fileAttributeContainsAnyOf("OECD Number","402","403","404","405","425","429","436","438","439","471","487")
&& (
text.contains("DATA REQUIREMENT")
|| text.contains("TEST GUIDELINE")
|| text.contains("MÉTODO(S) DE REFERÊNCIA(S):")
)
&& (
text.contains("OECD")
|| text.contains("EPA")
|| text.contains("OPPTS")
)
)
then
section.addFileAttribute("OECD Number", "(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", true, 1);
section.addFileAttribute("OECD Number", "(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1);
end
rule "1: Guidelines"
when
Section(
(
text.contains("DATA REQUIREMENT")
|| text.contains("TEST GUIDELINE")
|| text.contains("MÉTODO(S) DE REFERÊNCIA(S):")
)
&& (
text.contains("OECD")
|| text.contains("EPA")
|| text.contains("OPPTS")
)
)
then
section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", true, 1, "oecd_guideline_number", 1, "OECD Guideline no. found", "n-a");
section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
section.redactByRegEx("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", true, 1, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1, "oecd_guideline_number", 1, "OECD Guideline number found", "n-a");
section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a");
end
rule "2: Full Table extraction (Guideline Deviation)"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Full Table")
&& hasTableHeader("Sex")
)
then
section.redactSectionTextWithoutHeadLine("guideline_deviation",2,"Full table extraction into guideline deviation","n-a");
end
rule "3: Individual row extraction (Clinical Signs)"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Individual Rows")
&& hasTableHeader("Animal No.")
&& (rowEquals("Animal No.","120-2") || rowEquals("Animal No.","120-5"))
)
then
section.redactSectionTextWithoutHeadLine("clinical_signs",3,"Individual row based on animal number","n-a");
end
rule "4: Individual column extraction (Strain)"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Individual Column")
&& hasTableHeader("Sex")
)
then
section.redactCell("Sex",4,"dosages",false,"Individual column based on column header","n-a");
end
rule "5: Dose Mortality"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Combined Columns")
&& hasTableHeader("Mortality")
&& hasTableHeader("Dosage (mg/kg bw)")
)
then
section.redactCell("Mortality",5,"dose_mortality",false,"Dose Mortality found.","n-a");
section.redactCell("Dosage (mg/kg bw)",5,"dose_mortality_dose",false,"Dose Mortality dose found.","n-a");
end
rule "6: targeted cell extraction (Experimental Start date)"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Value Extraction")
&& hasTableHeader("Mortality")
&& (rowEquals("Sex","male") || rowEquals("Sex","Male"))
&& rowEquals("Mortality","Survived")
)
then
section.redactCell("Treatment start",6,"experimental_start_date",false,"Female deaths date to experimental start date","n-a");
end
rule "7: targeted cell extraction (Experimental Stop date)"
when
Section(
isInTable()
&& (searchText.contains("female") || searchText.contains("Female"))
&& searchText.contains("Survived")
)
then
section.redactCellBelow(7,"experimental_end_date",true,false,"Female deaths date to experimental start date","n-a", "Sex", "Group 2");
end
rule "8: Indicator (Species)"
when
Section(
fileAttributeByLabelEqualsIgnoreCase("OECD Number","425")
&& headlineContainsWord("Entity-Based")
&& matchesType("vertebrates")
)
then
section.redactCell("Title",8,"study_design",false,"Vertebrate study found","n-a");
end

473
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl
View File

@ -45,476 +45,59 @@ declare GuidelineMapping
guideline: String
end
//------------------------------------ Default Components rules ------------------------------------
//------------------------------------ Demo Components rules ------------------------------------
rule "StudyTitle.0.0: First Title found"
rule "FullTable.1.1: All rows of matching table"
when
$titleCandidates: List() from collect (Entity(type == "title"))
$tableRowValues: List() from collect (Entity(type == "full_table_row"))
then
componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, "");
componentCreationService.joiningFromSameTableRow("FullTable.1.1", "1.1 Full Table", $tableRowValues);
end
rule "PerformingLaboratory.1.0: Performing Laboratory name and country found in same section"
rule "IndivRowExtr.1.2: Individual rows of table"
when
$laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
$laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node)
not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset))
then
componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry));
end
rule "PerformingLaboratory.2.0: Performing Laboratory name but no country found in same section"
when
$laboratoryName: Entity(type == "laboratory_name", $node: containingNode)
not Entity(type == "laboratory_country", containingNode == $node)
then
componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName));
end
rule "PerformingLaboratory.0.2: Performing Laboratory not found"
salience -1
when
not Component(name == "Performing_Laboratory")
then
componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback");
end
rule "ReportNumber.0.0: First Report number found"
when
$reportNumberCandidates: List() from collect (Entity(type == "report_number"))
then
componentCreationService.firstOrElse("ReportNumber.0.0", "Report_Number", $reportNumberCandidates, "");
end
rule "GLPStudy.0.0: GLP Study found"
when
$glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study"))
then
componentCreationService.create("GLPStudy.0.0", "GLP_Study", "Yes", "Yes if present, No if not", $glpStudyList);
end
rule "GLPStudy.1.0: GLP Study not found"
when
not Entity(type == "glp_study")
then
componentCreationService.create("GLPStudy.1.0", "GLP_Study", "No", "Yes if present, No if not");
end
rule "TestGuideline.0.0: create OECD number and year guideline mappings"
salience 2
when
Entity(type == "oecd_guideline_number")
Entity(type == "oecd_guideline_year")
then
insert(new GuidelineMapping("425", "2008", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)"));
insert(new GuidelineMapping("425", "2001", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)"));
insert(new GuidelineMapping("402", "2017", "Nº 402: Acute Dermal Toxicity (09/10/2017)"));
insert(new GuidelineMapping("402", "1987", "Nº 402: Acute Dermal Toxicity (24/02/1987)"));
insert(new GuidelineMapping("403", "2009", "Nº 403: Acute Inhalation Toxicity (08/09/2009)"));
insert(new GuidelineMapping("403", "1981", "Nº 403: Acute Inhalation Toxicity (12/05/1981)"));
insert(new GuidelineMapping("433", "2018", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)"));
insert(new GuidelineMapping("433", "2017", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)"));
insert(new GuidelineMapping("436", "2009", "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)"));
insert(new GuidelineMapping("404", "1981", "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)"));
insert(new GuidelineMapping("404", "1992", "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)"));
insert(new GuidelineMapping("404", "2002", "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)"));
insert(new GuidelineMapping("404", "2015", "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)"));
insert(new GuidelineMapping("405", "2017", "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)"));
insert(new GuidelineMapping("405", "2012", "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)"));
insert(new GuidelineMapping("405", "2002", "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)"));
insert(new GuidelineMapping("405", "1987", "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)"));
insert(new GuidelineMapping("429", "2002", "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)"));
insert(new GuidelineMapping("429", "2010", "Nº 429: Skin Sensitisation (23/07/2010)"));
insert(new GuidelineMapping("442A", "2018", "Nº 442A: Skin Sensitization (23/07/2018)"));
insert(new GuidelineMapping("442B", "2018", "Nº 442B: Skin Sensitization (27/06/2018)"));
insert(new GuidelineMapping("471", "1997", "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)"));
insert(new GuidelineMapping("471", "2020", "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)"));
insert(new GuidelineMapping("406", "1992", "Nº 406: Skin Sensitisation (1992)"));
insert(new GuidelineMapping("428", "2004", "Nº 428: Split-Thickness Skin test (2004)"));
insert(new GuidelineMapping("438", "2018", "Nº 438: Eye Irritation (26/06/2018)"));
insert(new GuidelineMapping("439", "2019", "Nº 439: Skin Irritation (2019)"));
insert(new GuidelineMapping("474", "2016", "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)"));
insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)"));
end
rule "TestGuideline.0.1: match OECD number and year with guideline mappings"
salience 1
when
not Component(name == "Test_Guidelines_1")
GuidelineMapping($year: year, $number: number, $guideline: guideline)
$guidelineNumber: Entity(type == "oecd_guideline_number", value == $number)
$guidelineYear: Entity(type == "oecd_guideline_year", value == $year)
then
componentCreationService.create(
"TestGuideline.0.0",
"Test_Guidelines_1",
$guideline,
"OECD Number and guideline year mapped!",
List.of($guidelineNumber, $guidelineYear)
);
end
rule "TestGuideline.1.0: no guideline mapping found"
when
not Component(name == "Test_Guidelines_1")
$guideLine: Entity(type == "oecd_guideline")
then
componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine));
end
rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines"
when
$guidelines: List() from collect (Entity(type == "epa_guideline" || type == "ec_guideline"))
then
componentCreationService.joining("TestGuideline.2.0", "Test_Guidelines_2", $guidelines);
end
rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy"
when
$startDates: List() from collect (Entity(type == "experimental_start_date"))
then
componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates);
end
rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy"
when
$endDates: List() from collect (Entity(type == "experimental_end_date"))
then
componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates);
end
rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification"
when
$batchNumbers: List() from collect (Entity(type == "batch_number"))
then
componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers);
end
rule "StudyConclusion.0.0: Study conclusion in first found section"
when
$oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$studyConclusions: List() from collect(Entity(type == "study_conclusion"))
then
componentCreationService.joiningFromFirstSectionOnly("StudyConclusion.0.0", "Study_Conclusion", $studyConclusions, " ");
end
rule "GuidelineDeviation.0.0: Guideline deviation as sentences"
when
$oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$guidelineDeviations: List() from collect (Entity(type == "guideline_deviation"))
then
componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n");
end
rule "Species.0.0: First found species"
when
$oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$species: List() from collect (Entity(type == "species"))
then
componentCreationService.firstOrElse("Species.0.0", "Species", $species, "");
end
rule "Strain.0.0: First found strain"
when
$oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$strain: List() from collect (Entity(type == "strain"))
then
componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, "");
end
rule "Conclusion.0.0: Unique values of Conclusion LD50"
when
$oecdNumber: String() from List.of("402", "403", "425", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$conclusions: List() from collect (Entity(type == "ld50_value"))
then
componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions);
end
rule "Conclusion0.1.0: Greater than found"
when
$oecdNumber: String() from List.of("402", "403", "425", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater"))
then
componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater than", "Entity of type 'ld50_greater' found", $conclusions);
end
rule "Conclusion.1.1: Greater than not found"
when
$oecdNumber: String() from List.of("402", "403", "425", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
not Entity(type == "ld50_greater")
then
componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No entity of type 'ld50_greater' found");
end
rule "Conclusion.2.0: Minimum confidence as unique values"
when
$oecdNumber: String() from List.of("402", "403", "425", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$conclusions: List() from collect (Entity(type == "confidence_minimal"))
then
componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions);
end
rule "Conclusion.3.0: Maximum confidence as unique values"
when
$oecdNumber: String() from List.of("402", "403", "425", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$conclusions: List() from collect (Entity(type == "confidence_maximal"))
then
componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions);
end
rule "Necropsy.0.0: Necropsy findings from longest section"
when
FileAttribute(label == "OECD Number", value == "402")
$necropsies: List() from collect (Entity(type == "necropsy_findings"))
then
componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " ");
end
rule "Necropsy.0.1: Necropsy findings joined with \n"
when
FileAttribute(label == "OECD Number", value == "403" || value == "436")
$necropsies: List() from collect (Entity(type == "necropsy_findings"))
then
componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n");
end
rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block"
when
FileAttribute(label == "OECD Number", value == "402")
$dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)"))
then
componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " ");
end
rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block"
when
$oecdNumber: String() from List.of("403", "436")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$exposures: List() from collect (Entity(type == "4h_exposure"))
then
componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " ");
end
rule "StudyDesign.0.0: Study design as one block"
when
$oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$studyDesigns: List() from collect (Entity(type == "study_design"))
then
componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " ");
end
rule "Results.0.0: Results and conclusions as joined values"
when
$oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "results_and_conclusion"))
then
componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " ");
end
rule "WeightBehavior.0.0: Weight change behavior as sentences"
when
FileAttribute(label == "OECD Number", value == "402")
$weightChanges: List() from collect (Entity(type == "weight_behavior_changes"))
then
componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n");
end
rule "MortalityStatement.0.0: Mortality statements as one block"
when
FileAttribute(label == "OECD Number", value == "402")
$mortalityStatements: List() from collect (Entity(type == "mortality_statement"))
then
componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " ");
end
rule "ClinicalObservations.0.0: Clinical observations as sentences"
when
FileAttribute(label == "OECD Number", value == "403")
$observations: List() from collect (Entity(type == "clinical_observations"))
then
componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n");
end
rule "BodyWeight.0.0: Bodyweight changes as sentences"
when
FileAttribute(label == "OECD Number", value == "403")
$weightChanges: List() from collect (Entity(type == "bodyweight_changes"))
then
componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n");
end
rule "Detailing.0.0: Detailing of reported changes as one block"
when
$oecdNumber: String() from List.of("404", "405")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$detailings: List() from collect (Entity(type == "detailing"))
then
componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " ");
end
rule "Sex.0.0: Male sex found"
when
$oecdNumber: String() from List.of("405", "429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$males: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "male" || value.toLowerCase() == "males")))
then
componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males);
end
rule "Sex.1.0: Female sex found"
when
$oecdNumber: String() from List.of("405", "429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$females: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "female" || value.toLowerCase() == "females")))
then
componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females);
end
rule "NumberOfAnimals.0.0: Number of animals found"
when
$oecdNumber: String() from List.of("405", "429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$numberOfAnimals: Entity(type == "number_of_animals")
then
componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals);
end
rule "NumberOfAnimals.1.0: Count unique occurences of animals"
when
$oecdNumber: String() from List.of("405", "429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
not Entity(type == "number_of_animals")
$animals: List() from collect (Entity(type == "animal_number"))
then
componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals);
end
rule "ClinicalSigns.0.0: Clinical signs as sentences"
when
$oecdNumber: String() from List.of("425")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$clinicalSigns: List() from collect (Entity(type == "clinical_signs"))
then
componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n");
componentCreationService.joiningFromSameTableRow("IndivRowExtr.1.2", "1.2 Individual Rows", $clinicalSigns);
end
rule "DoseMortality.0.0: Dose mortality joined with dose from same table row"
rule "IndivColExtr.1.3: Individual column of table"
when
$tableColValues: List() from collect (Entity(type == "dosages"))
then
componentCreationService.joiningFromSameTableRow("IndivColExtr.1.3", "1.3 Individual Column", $tableColValues);
end
rule "CombColExtr.2.1: Combined Columns Extraction"
when
$oecdNumber: String() from List.of("425")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose"))
then
componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities);
componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "2.1 Combined Columns", $doseMortalities);
end
rule "Mortality.0.0: Mortality as one block"
rule "ValueExtrRef.2.2: Cells containing dose for survived males"
when
$oecdNumber: String() from List.of("425")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$mortalities: List() from collect (Entity(type == "mortality"))
$tableValues: List() from collect (Entity(type == "doses_mg_kg_bw"))
then
componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " ");
componentCreationService.joining("ValueExtrRef.2.2", "2.2 Individual Table Values", $tableValues);
end
rule "Dosages.0.0: First found value of Dosages"
rule "AdvTableExtr.2.3: Cells containing dose for survived males"
when
$oecdNumber: String() from List.of("425")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$mortalities: List() from collect (Entity(type == "dosages"))
$tableValues: List() from collect (Entity(type == "experiment_female_survived"))
then
componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, "");
componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
end
rule "PrelimResults.0.0: Preliminary test results as sentences"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "preliminary_test_results"))
then
componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n");
end
rule "TestResults.0.0: Test results as one block"
rule "EntityBasedExtr.2.4: Cells containing dose for survived males"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "test_results"))
$tableValues: List() from collect (Entity(type == "study_design"))
then
componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " ");
componentCreationService.joiningFromSameTableRow("EntityBasedExtr.2.4", "2.4 Entity-Based Values", $tableValues);
end
rule "PositiveControl.0.0: Was the definitive study conducted with positive control"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "positive_control"))
then
componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " ");
end
rule "MainResults.0.0: Results from main study as one block"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List() from collect (Entity(type == "results_(main_study)"))
then
componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " ");
end
rule "UsedApproach.0.0: Used approach found and mapped to 'Group'"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
$results: List(!isEmpty()) from collect (Entity(type == "approach_used"))
then
componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results);
end
rule "UsedApproach.1.0: Used approach not found and thus 'Individual'"
when
$oecdNumber: String() from List.of("429")
FileAttribute(label == "OECD Number", value == $oecdNumber)
not Entity(type == "approach_used")
then
componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'");
end
/*
rule "DefaultComponents.999.0: Create components for all unmapped entities."
salience -999
when
$allEntities: List(!isEmpty()) from collect (Entity())
then
componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities);
end
*/
//------------------------------------ Component merging rules ------------------------------------
/*
rule "X.0.0: merge duplicate component references"
when
$first: Component()
$duplicate: Component(this != $first, name == $first.name, value == $first.value)
then
$first.getReferences().addAll($duplicate.getReferences());
retract($duplicate);
end
*/

55
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl Normal file
View File

@ -0,0 +1,55 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.model.component.Component;
import com.iqser.red.service.redaction.v1.server.model.component.Entity;
import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
global ComponentCreationService componentCreationService
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
query "getComponents"
$component: Component()
end
//------------------------------------ Guideline mapping object ------------------------------------
declare GuidelineMapping
number: String
year: String
guideline: String
end
//------------------------------------ Test Components rules ------------------------------------
rule "AdvTableExtr.2.3: Cells containing dose for survived males"
when
$tableValues: List() from collect (Entity(type == "test"))
then
componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues);
end

401
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl Normal file
View File

@ -0,0 +1,401 @@
package drools
import static java.lang.String.format;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
import java.util.List;
import java.util.LinkedList;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Collection;
import java.util.stream.Stream;
import java.util.Optional;
import com.iqser.red.service.redaction.v1.server.model.document.*;
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus;
global Document document
global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999
when
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
then
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
.forEach(entity -> {
Collection<MatchedRule> matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue());
entity.addMatchedRules(matchedRules);
});
end
// --------------------------------------- Your rules below this line --------------------------------------------------
rule "TAB.0.0: Study Type File Attribute"
when
not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487"))
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):")
&& containsAnyString("OECD", "EPA", "OPPTS"))
then
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream()
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
.forEach(fileAttribute -> insert(fileAttribute));
end
rule "TAB.0.1: Guidelines"
when
$section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
then
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found"));
entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found"));
entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section)
.forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found"));
end
rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)"
when
$section: Section(containsString("Maximum occurrence"))
$table: Table() from $section.streamChildren().toList()
TableCell(containsWordIgnoreCase("water"), $row: row) from $table.streamTableCells().toList()
$test: TableCell($row == row) from $table.streamTableCells().toList()
then
System.out.println("AAAA: " + $test);
entityCreationService.bySemanticNode($test, "test", EntityType.ENTITY)
.ifPresent(entity -> entity.apply("TAB.6.0", "Some test stuff"));
end
//------------------------------------ Manual redaction rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$entityToBeResized: TextEntity(matchesAnnotationId($id))
then
manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($entityToBeResized);
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.0.1: Apply manual resize redaction"
salience 128
when
$resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
$imageToBeResized: Image(id == $id)
then
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
retract($resizeRedaction);
update($imageToBeResized);
update($imageToBeResized.getParent());
end
// Rule unit: MAN.1
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
then
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($entityToBeRemoved);
retract($idRemoval);
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
end
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
salience 128
when
$idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED)
$imageEntityToBeRemoved: Image($id == id)
then
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
update($imageEntityToBeRemoved);
retract($idRemoval);
update($imageEntityToBeRemoved.getParent());
end
// Rule unit: MAN.2
rule "MAN.2.0: Apply force redaction"
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToForce: TextEntity(matchesAnnotationId($id))
then
$entityToForce.getManualOverwrite().addChange($force);
update($entityToForce);
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
retract($force);
end
rule "MAN.2.1: Apply force redaction to images"
salience 128
when
$force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToForce: Image(id == $id)
then
$imageToForce.getManualOverwrite().addChange($force);
update($imageToForce);
update($imageToForce.getParent());
retract($force);
end
// Rule unit: MAN.3
rule "MAN.3.0: Apply entity recategorization"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type)
then
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization);
retract($recategorization);
// Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication.
retract($entityToBeRecategorized);
end
rule "MAN.3.1: Apply entity recategorization of same type"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type)
then
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
retract($recategorization);
end
rule "MAN.3.2: Apply image recategorization"
salience 128
when
$recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate)
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
$imageToBeRecategorized: Image($id == id)
then
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
update($imageToBeRecategorized);
update($imageToBeRecategorized.getParent());
retract($recategorization);
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
end
rule "MAN.4.1: Apply legal basis change"
salience 128
when
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
end
//------------------------------------ Entity merging rules ------------------------------------
// Rule unit: X.0
rule "X.0.0: Remove Entity contained by Entity of same type"
salience 65
when
$larger: TextEntity($type: type, $entityType: entityType, active())
$contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active())
then
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
retract($contained);
end
// Rule unit: X.1
rule "X.1.0: Merge intersecting Entities of same type"
salience 64
when
$first: TextEntity($type: type, $entityType: entityType, !resized(), active())
$second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
then
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
$first.remove("X.1.0", "merge intersecting Entities of same type");
$second.remove("X.1.0", "merge intersecting Entities of same type");
retract($first);
retract($second);
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
// Rule unit: X.3
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
salience 64
when
$falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active())
$recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
retract($recommendation);
end
// Rule unit: X.4
rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"
salience 256
when
$entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$entity.addEngines($recommendation.getEngines());
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type");
retract($recommendation);
end
// Rule unit: X.5
rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY"
salience 256
when
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active())
then
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY");
retract($recommendation);
end
// Rule unit: X.6
rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY"
salience 32
when
$higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active())
then
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY");
retract($lowerRank);
end
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");
retract($higherRank);
end
// Rule unit: X.7
rule "X.7.0: remove all images"
salience 512
when
$image: Image(imageType != ImageType.OCR, !hasManualChanges())
then
$image.remove("X.7.0", "remove all images");
retract($image);
end
//------------------------------------ File attributes rules ------------------------------------
// Rule unit: FA.1
rule "FA.1.0: Remove duplicate FileAttributes"
salience 64
when
$fileAttribute: FileAttribute($label: label, $value: value)
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
then
retract($duplicate);
end