diff --git a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts index cc87a214..5ecbd3cb 100644 --- a/redaction-service-v1/redaction-service-api-v1/build.gradle.kts +++ b/redaction-service-v1/redaction-service-api-v1/build.gradle.kts @@ -7,7 +7,7 @@ description = "redaction-service-api-v1" dependencies { implementation("org.springframework:spring-web:6.0.12") - implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.207.0") + implementation("com.iqser.red.service:persistence-service-internal-api-v1:2.240.0") } publishing { diff --git a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java index 98d1194e..fcc8b834 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java +++ b/redaction-service-v1/redaction-service-server-v1/src/main/java/com/iqser/red/service/redaction/v1/server/service/document/ComponentCreationService.java @@ -19,6 +19,7 @@ import org.kie.api.runtime.KieSession; import com.iqser.red.service.redaction.v1.server.model.component.Component; import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; @@ -102,7 +103,11 @@ public class ComponentCreationService { referencedEntities.addAll(references); - kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name).value(value).valueDescription(valueDescription) + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .name(name) + .value(value) + .valueDescription(valueDescription) .references(new LinkedList<>(references)) .build()); } @@ -293,6 +298,30 @@ public class ComponentCreationService { } + /** + * Counts the distinct rows where any of the provided entities occur and creates a component for each distinct table with its respective row count. + * + * @param ruleIdentifier the identifier of the rule + * @param name the name of the record + * @param entities the collection of entities to count row values from + */ + public void rowValueCount(String ruleIdentifier, String name, Collection entities) { + + entities.stream().collect(Collectors.groupingBy(this::getFirstTable)).forEach((optionalTable, groupedEntities) -> { + + if (optionalTable.isEmpty()) { + return; + } + + long count = groupedEntities.stream() + .collect(Collectors.groupingBy(entity -> getFirstTableCell(entity).map(TableCell::getRow).orElse(-1))) + .size(); + + create(ruleIdentifier, name, String.valueOf(count), "Count rows with values in the entity references in same table", entities); + }); + } + + /** * Creates a component for each sentence in the collection of entities. * @@ -354,7 +383,9 @@ public class ComponentCreationService { */ public void createComponentsForUnMappedEntities(String ruleIdentifier, Collection entities) { - entities.stream().filter(entity -> !referencedEntities.contains(entity)).sorted(EntityComparators.first()) + entities.stream() + .filter(entity -> !referencedEntities.contains(entity)) + .sorted(EntityComparators.first()) .forEach(entity -> create(ruleIdentifier, entity.getType(), entity.getValue(), "Unmapped Entity", List.of(entity))); } @@ -429,7 +460,8 @@ public class ComponentCreationService { .sorted(Comparator.comparingInt(Map.Entry::getKey)) .map(Map.Entry::getValue) .forEach(entitiesInSameRow -> create(ruleIdentifier, - name, entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")), + name, + entitiesInSameRow.stream().sorted(EntityComparators.first()).map(Entity::getValue).collect(Collectors.joining(", ")), valueDescription, entitiesInSameRow)); }); @@ -450,6 +482,20 @@ public class ComponentCreationService { } + private Optional getFirstTableCell(Entity entity) { + + SemanticNode node = entity.getContainingNode(); + while (!(node instanceof TableCell)) { + if (!node.hasParent()) { + return Optional.empty(); + } + node = node.getParent(); + } + + return Optional.of((TableCell) node); + } + + /** * Creates a new component with the given rule identifier, name, value, and value description. * If the component is part of a table, it also takes a list of entities that belong to the same table row. @@ -474,7 +520,9 @@ public class ComponentCreationService { */ public void create(String ruleIdentifier, String name, String value) { - kieSession.insert(Component.builder().matchedRule(RuleIdentifier.fromString(ruleIdentifier)).name(name) + kieSession.insert(Component.builder() + .matchedRule(RuleIdentifier.fromString(ruleIdentifier)) + .name(name) .value(value) .valueDescription("") .references(Collections.emptyList()) diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java index 87c73e96..73481060 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java +++ b/redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/AnalysisTest.java @@ -43,33 +43,30 @@ import com.knecon.fforesight.tenantcommons.TenantContext; @Import(AnalysisTest.RedactionIntegrationTestConfiguration.class) public class AnalysisTest extends AbstractRedactionIntegrationTest { - private static final String RULES = loadFromClassPath("drools/table_demo.drl"); - private static final String COMPONENT_RULES = loadFromClassPath("drools/table_demo_components.drl"); + private static final String RULES = loadFromClassPath("drools/test_rules.drl"); + private static final String COMPONENT_RULES = loadFromClassPath("drools/test_components.drl"); @Test @Disabled - public void analyzeTableDemoFile() { + public void analyzeTableDemoFile() throws IOException { - AnalyzeRequest request = uploadFileToStorage("files/TableDemo/Table_examples.pdf"); + AnalyzeRequest request = uploadFileToStorage("files/Metolachlor/S-Metolachlor_RAR_01_Volume_1_2018-09-06.pdf"); System.out.println("Start Full integration test"); analyzeDocumentStructure(LayoutParsingType.DOCUMINE, request); System.out.println("Finished structure analysis"); AnalyzeResult result = analyzeService.analyze(request); System.out.println("Finished analysis"); -// var redactionLog = redactionStorageService.getRedactionLog(TEST_DOSSIER_ID, TEST_FILE_ID); -// var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID); + var componentLog = redactionStorageService.getComponentLog(TEST_DOSSIER_ID, TEST_FILE_ID); AnnotateResponse annotateResponse = annotationService.annotate(AnnotateRequest.builder().dossierId(TEST_DOSSIER_ID).fileId(TEST_FILE_ID).build()); String outputFileName = OsUtils.getTemporaryDirectory() + "/Documine.pdf"; try (FileOutputStream fileOutputStream = new FileOutputStream(outputFileName)) { fileOutputStream.write(annotateResponse.getDocument()); - } catch (IOException e) { - throw new RuntimeException(e); } - } diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl index 69015695..7c54444f 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl @@ -53,7 +53,7 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; -import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; global Document document global EntityCreationService entityCreationService @@ -65,7 +65,23 @@ global Dictionary dictionary query "getFileAttributes" $fileAttribute: FileAttribute() end -//--------------------------------------------------------------------------- + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LocalDictionarySearch.0 +rule "LDS.0.0: Run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end +// --------------------------------------- Your rules below this line -------------------------------------------------- rule "TAB.0.0: Study Type File Attribute" when @@ -106,7 +122,7 @@ rule "TAB.1.0: Full Table extraction (Guideline Deviation)" $table: Table() from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList() $tableCell: TableCell(!header) from $table.streamTableCells().toList() then - entityCreationService.bySemanticNode($tableCell, "guideline_deviation", EntityType.ENTITY) + entityCreationService.bySemanticNode($tableCell, "full_table_row", EntityType.ENTITY) .ifPresent(entity -> entity.apply("TAB.1.0", "full table extracted")); end @@ -139,28 +155,26 @@ rule "TAB.4.0: Combined Columns Extraction - Sex and Dosage" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("425")) $section: Section(getHeadline().containsString("Combined Columns")) - $table: Table(hasHeader("Sex"), hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList() - $maleCells: TableCell($row: row, containsAnyWordIgnoreCase("Male")) from $table.streamTableCellsWithHeader("Sex").toList() - $dosageCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Dosage").toList() + $table: Table(hasHeader("Dosage (mg/kg bw)")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList() then - entityCreationService.bySemanticNode($maleCells, "combined_male_dosage", EntityType.ENTITY) - .ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male")); - entityCreationService.bySemanticNode($dosageCells, "combined_male_dosage", EntityType.ENTITY) - .ifPresent(entity -> entity.apply("TAB.4.0", "Dosage combined in row with male")); + $table.streamTableCellsWithHeader("Dosage (mg/kg bw)") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality_dose", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.apply("TAB.4.0", "Dose Mortality dose found.")); end rule "TAB.4.1: Combined Columns Extraction - Sex and Mortality" when FileAttribute(label == "OECD Number", valueEqualsAnyOf("425")) $section: Section(getHeadline().containsString("Combined Columns")) - $table: Table(hasHeader("Sex"), hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList() - $femaleCells: TableCell($row: row, containsAnyWordIgnoreCase("Female")) from $table.streamTableCellsWithHeader("Sex").toList() - $mortalityCells: TableCell($row == row) from $table.streamTableCellsWithHeader("Mortality").toList() + $table: Table(hasHeader("Mortality")) from $section.getParent().streamAllSubNodesOfType(NodeType.TABLE).toList() then - entityCreationService.bySemanticNode($femaleCells, "combined_female_mortality", EntityType.ENTITY) - .ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female")); - entityCreationService.bySemanticNode($mortalityCells, "combined_female_mortality", EntityType.ENTITY) - .ifPresent(entity -> entity.apply("TAB.4.1", "Mortality combined in row with female")); + $table.streamTableCellsWithHeader("Mortality") + .map(tableCell -> entityCreationService.bySemanticNode(tableCell, "dose_mortality", EntityType.ENTITY)) + .filter(Optional::isPresent) + .map(Optional::get) + .forEach(redactionEntity -> redactionEntity.apply("TAB.4.1", "Dose Mortality found.")); end rule "TAB.5.0: Targeted cell extraction" @@ -194,7 +208,7 @@ rule "TAB.7.0: Indicator (Species)" $section: Section(getHeadline().containsString("Entity-Based")) $table: Table() from $section.streamAllSubNodesOfType(NodeType.TABLE).toList() TableCell(isHeader(), containsString("Title"), $col: col) from $table.streamTableCells().toList() - TableCell(hasEntitiesOfType("vertebrate"), $row: row) from $table.streamTableCells().toList() + TableCell(hasEntitiesOfType("vertebrates"), $row: row) from $table.streamTableCells().toList() $cell: TableCell($col == col, $row == row) from $table.streamTableCells().toList() then entityCreationService.bySemanticNode($cell, "study_design", EntityType.ENTITY) @@ -407,6 +421,31 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY" end +// Rule unit: X.6 +rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" + salience 32 + when + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + then + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); + retract($lowerRank); + end + + +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + + // Rule unit: X.7 rule "X.7.0: remove all images" salience 512 @@ -430,19 +469,3 @@ rule "FA.1.0: Remove duplicate FileAttributes" retract($duplicate); end - -//------------------------------------ Local dictionary search rules ------------------------------------ - -// Rule unit: LDS.0 -rule "LDS.0.0: Run local dictionary search" - agenda-group "LOCAL_DICTIONARY_ADDS" - salience -999 - when - $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() - then - entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) - .forEach(entity -> { - Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); - entity.addMatchedRules(matchedRules); - }); - end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_OLD.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_OLD.drl deleted file mode 100644 index 02dd3b78..00000000 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_OLD.drl +++ /dev/null @@ -1,134 +0,0 @@ -package drools - -import com.iqser.red.service.redaction.v1.server.redaction.model.Section - -global Section section - - -// --------------------------------------- Your rules below this line-------------------------------------------------- - -rule "0a: Study Type File Attribute" - when - Section( - !fileAttributeContainsAnyOf("OECD Number","402","403","404","405","425","429","436","438","439","471","487") - && ( - text.contains("DATA REQUIREMENT") - || text.contains("TEST GUIDELINE") - || text.contains("MÉTODO(S) DE REFERÊNCIA(S):") - ) - && ( - text.contains("OECD") - || text.contains("EPA") - || text.contains("OPPTS") - ) - ) - then - section.addFileAttribute("OECD Number", "(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", true, 1); - section.addFileAttribute("OECD Number", "(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1); - end - - -rule "1: Guidelines" - when - Section( - ( - text.contains("DATA REQUIREMENT") - || text.contains("TEST GUIDELINE") - || text.contains("MÉTODO(S) DE REFERÊNCIA(S):") - ) - && ( - text.contains("OECD") - || text.contains("EPA") - || text.contains("OPPTS") - ) - ) - then - section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", true, 1, "oecd_guideline_number", 1, "OECD Guideline no. found", "n-a"); - section.redactByRegEx("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a"); - section.redactByRegEx("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", true, 1, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a"); - section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 1, "oecd_guideline_number", 1, "OECD Guideline number found", "n-a"); - section.redactByRegEx("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", true, 2, "oecd_guideline_year", 1, "OECD Guideline year found", "n-a"); - end - - -rule "2: Full Table extraction (Guideline Deviation)" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Full Table") - && hasTableHeader("Sex") - ) - then - section.redactSectionTextWithoutHeadLine("guideline_deviation",2,"Full table extraction into guideline deviation","n-a"); - end - -rule "3: Individual row extraction (Clinical Signs)" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Individual Rows") - && hasTableHeader("Animal No.") - && (rowEquals("Animal No.","120-2") || rowEquals("Animal No.","120-5")) - ) - then - section.redactSectionTextWithoutHeadLine("clinical_signs",3,"Individual row based on animal number","n-a"); - end - -rule "4: Individual column extraction (Strain)" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Individual Column") - && hasTableHeader("Sex") - ) - then - section.redactCell("Sex",4,"dosages",false,"Individual column based on column header","n-a"); - end - -rule "5: Dose Mortality" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Combined Columns") - && hasTableHeader("Mortality") - && hasTableHeader("Dosage (mg/kg bw)") - ) - then - section.redactCell("Mortality",5,"dose_mortality",false,"Dose Mortality found.","n-a"); - section.redactCell("Dosage (mg/kg bw)",5,"dose_mortality_dose",false,"Dose Mortality dose found.","n-a"); - end - -rule "6: targeted cell extraction (Experimental Start date)" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Value Extraction") - && hasTableHeader("Mortality") - && (rowEquals("Sex","male") || rowEquals("Sex","Male")) - && rowEquals("Mortality","Survived") - ) - then - section.redactCell("Treatment start",6,"experimental_start_date",false,"Female deaths date to experimental start date","n-a"); - end - -rule "7: targeted cell extraction (Experimental Stop date)" - when - Section( - isInTable() - && (searchText.contains("female") || searchText.contains("Female")) - && searchText.contains("Survived") - ) - then - section.redactCellBelow(7,"experimental_end_date",true,false,"Female deaths date to experimental start date","n-a", "Sex", "Group 2"); - end - -rule "8: Indicator (Species)" - when - Section( - fileAttributeByLabelEqualsIgnoreCase("OECD Number","425") - && headlineContainsWord("Entity-Based") - && matchesType("vertebrates") - ) - then - section.redactCell("Title",8,"study_design",false,"Vertebrate study found","n-a"); - end \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl index f1d18f26..d3e5c6ff 100644 --- a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo_components.drl @@ -45,476 +45,59 @@ declare GuidelineMapping guideline: String end -//------------------------------------ Default Components rules ------------------------------------ +//------------------------------------ Demo Components rules ------------------------------------ -rule "StudyTitle.0.0: First Title found" +rule "FullTable.1.1: All rows of matching table" when - $titleCandidates: List() from collect (Entity(type == "title")) + $tableRowValues: List() from collect (Entity(type == "full_table_row")) then - componentCreationService.firstOrElse("StudyTitle.0.0", "Study_Title", $titleCandidates, ""); + componentCreationService.joiningFromSameTableRow("FullTable.1.1", "1.1 Full Table", $tableRowValues); end -rule "PerformingLaboratory.1.0: Performing Laboratory name and country found in same section" +rule "IndivRowExtr.1.2: Individual rows of table" when - $laboratoryName: Entity(type == "laboratory_name", $node: containingNode) - $laboratoryCountry: Entity(type == "laboratory_country", containingNode == $node) - not Entity(type == "laboratory_country", containingNode == $node, Math.abs($laboratoryName.startOffset - startOffset) < Math.abs($laboratoryName.startOffset - $laboratoryCountry.startOffset)) - then - componentCreationService.create("PerformingLaboratory.1.0", "Performing_Laboratory", $laboratoryName.getValue() + ", " + $laboratoryCountry.getValue(), "Laboratory name and country found!", List.of($laboratoryName, $laboratoryCountry)); - end - -rule "PerformingLaboratory.2.0: Performing Laboratory name but no country found in same section" - when - $laboratoryName: Entity(type == "laboratory_name", $node: containingNode) - not Entity(type == "laboratory_country", containingNode == $node) - then - componentCreationService.create("PerformingLaboratory.2.0", "Performing_Laboratory", $laboratoryName.getValue(), "Only laboratory name found!", List.of($laboratoryName)); - end - -rule "PerformingLaboratory.0.2: Performing Laboratory not found" - salience -1 - when - not Component(name == "Performing_Laboratory") - then - componentCreationService.create("PerformingLaboratory.0.2", "Performing_Laboratory", "", "fallback"); - end - - -rule "ReportNumber.0.0: First Report number found" - when - $reportNumberCandidates: List() from collect (Entity(type == "report_number")) - then - componentCreationService.firstOrElse("ReportNumber.0.0", "Report_Number", $reportNumberCandidates, ""); - end - - -rule "GLPStudy.0.0: GLP Study found" - when - $glpStudyList: List(!isEmpty) from collect(Entity(type == "glp_study")) - then - componentCreationService.create("GLPStudy.0.0", "GLP_Study", "Yes", "Yes if present, No if not", $glpStudyList); - end - -rule "GLPStudy.1.0: GLP Study not found" - when - not Entity(type == "glp_study") - then - componentCreationService.create("GLPStudy.1.0", "GLP_Study", "No", "Yes if present, No if not"); - end - - -rule "TestGuideline.0.0: create OECD number and year guideline mappings" - salience 2 - when - Entity(type == "oecd_guideline_number") - Entity(type == "oecd_guideline_year") - then - insert(new GuidelineMapping("425", "2008", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (03/10/2008)")); - insert(new GuidelineMapping("425", "2001", "Nº 425: Acute oral Toxicity - Up-and-Down Procedure (17/12/2001)")); - insert(new GuidelineMapping("402", "2017", "Nº 402: Acute Dermal Toxicity (09/10/2017)")); - insert(new GuidelineMapping("402", "1987", "Nº 402: Acute Dermal Toxicity (24/02/1987)")); - insert(new GuidelineMapping("403", "2009", "Nº 403: Acute Inhalation Toxicity (08/09/2009)")); - insert(new GuidelineMapping("403", "1981", "Nº 403: Acute Inhalation Toxicity (12/05/1981)")); - insert(new GuidelineMapping("433", "2018", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (27/06/2018)")); - insert(new GuidelineMapping("433", "2017", "Nº 433: Acute Inhalation Toxicity: Fixed Concentration Procedure (09/10/2017)")); - insert(new GuidelineMapping("436", "2009", "Nº 436: Acute Inhalation Toxicity – Acute Toxic Class Method (08/09/2009)")); - insert(new GuidelineMapping("404", "1981", "Nº 404: Acute Dermal Irritation/Corrosion (12/05/1981)")); - insert(new GuidelineMapping("404", "1992", "Nº 404: Acute Dermal Irritation/Corrosion (17/07/1992)")); - insert(new GuidelineMapping("404", "2002", "Nº 404: Acute Dermal Irritation/Corrosion (24/04/2002)")); - insert(new GuidelineMapping("404", "2015", "Nº 404: Acute Dermal Irritation/Corrosion (28/07/2015)")); - insert(new GuidelineMapping("405", "2017", "Nº 405: Acute Eye Irritation/Corrosion (09/10/2017)")); - insert(new GuidelineMapping("405", "2012", "Nº 405: Acute Eye Irritation/Corrosion (02/10/2012)")); - insert(new GuidelineMapping("405", "2002", "Nº 405: Acute Eye Irritation/Corrosion (24/04/2002)")); - insert(new GuidelineMapping("405", "1987", "Nº 405: Acute Eye Irritation/Corrosion (24/02/1987)")); - insert(new GuidelineMapping("429", "2002", "Nº 429: Skin Sensitisation: Local Lymph Node Assay (24/04/2002)")); - insert(new GuidelineMapping("429", "2010", "Nº 429: Skin Sensitisation (23/07/2010)")); - insert(new GuidelineMapping("442A", "2018", "Nº 442A: Skin Sensitization (23/07/2018)")); - insert(new GuidelineMapping("442B", "2018", "Nº 442B: Skin Sensitization (27/06/2018)")); - insert(new GuidelineMapping("471", "1997", "Nº 471: Bacterial Reverse Mutation Test (21/07/1997)")); - insert(new GuidelineMapping("471", "2020", "Nº 471: Bacterial Reverse Mutation Test (26/06/2020)")); - insert(new GuidelineMapping("406", "1992", "Nº 406: Skin Sensitisation (1992)")); - insert(new GuidelineMapping("428", "2004", "Nº 428: Split-Thickness Skin test (2004)")); - insert(new GuidelineMapping("438", "2018", "Nº 438: Eye Irritation (26/06/2018)")); - insert(new GuidelineMapping("439", "2019", "Nº 439: Skin Irritation (2019)")); - insert(new GuidelineMapping("474", "2016", "Nº 474: Micronucleus Bone Marrow Cells Rat (2016)")); - insert(new GuidelineMapping("487", "2016", "Nº 487: Micronucleus Human Lymphocytes (2016)")); - end - -rule "TestGuideline.0.1: match OECD number and year with guideline mappings" - salience 1 - when - not Component(name == "Test_Guidelines_1") - GuidelineMapping($year: year, $number: number, $guideline: guideline) - $guidelineNumber: Entity(type == "oecd_guideline_number", value == $number) - $guidelineYear: Entity(type == "oecd_guideline_year", value == $year) - then - componentCreationService.create( - "TestGuideline.0.0", - "Test_Guidelines_1", - $guideline, - "OECD Number and guideline year mapped!", - List.of($guidelineNumber, $guidelineYear) - ); - end - -rule "TestGuideline.1.0: no guideline mapping found" - when - not Component(name == "Test_Guidelines_1") - $guideLine: Entity(type == "oecd_guideline") - then - componentCreationService.create("TestGuideline.2.0", "Test_Guidelines_1", $guideLine.getValue(), "No Mapping for OECD number and year found, using fallback instead!", List.of($guideLine)); - end - -rule "TestGuideline.2.0: All values of EPA guideline and EC guidelines" - when - $guidelines: List() from collect (Entity(type == "epa_guideline" || type == "ec_guideline")) - then - componentCreationService.joining("TestGuideline.2.0", "Test_Guidelines_2", $guidelines); - end - - -rule "StartDate.0.0: All experimental start dates converted to dd/MM/yyyy" - when - $startDates: List() from collect (Entity(type == "experimental_start_date")) - then - componentCreationService.convertDates("StartDate.0.0", "Experimental_Starting_Date", $startDates); - end - - -rule "CompletionDate.0.0: All experimental end dates converted to dd/MM/yyyy" - when - $endDates: List() from collect (Entity(type == "experimental_end_date")) - then - componentCreationService.convertDates("CompletionDate.0.0", "Experimental_Completion_Date", $endDates); - end - - -rule "AnalysisCertificate.0.0: Unique values of certificate of analysis batch identification" - when - $batchNumbers: List() from collect (Entity(type == "batch_number")) - then - componentCreationService.joiningUnique("AnalysisCertificate.0.0", "Certificate_of_Analysis_Batch_Identification", $batchNumbers); - end - -rule "StudyConclusion.0.0: Study conclusion in first found section" - when - $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $studyConclusions: List() from collect(Entity(type == "study_conclusion")) - then - componentCreationService.joiningFromFirstSectionOnly("StudyConclusion.0.0", "Study_Conclusion", $studyConclusions, " "); - end - -rule "GuidelineDeviation.0.0: Guideline deviation as sentences" - when - $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $guidelineDeviations: List() from collect (Entity(type == "guideline_deviation")) - then - componentCreationService.joining("GuidelineDeviation.0.0", "Deviation_from_the_Guideline", $guidelineDeviations, "\n"); - end - -rule "Species.0.0: First found species" - when - $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $species: List() from collect (Entity(type == "species")) - then - componentCreationService.firstOrElse("Species.0.0", "Species", $species, ""); - end - -rule "Strain.0.0: First found strain" - when - $oecdNumber: String() from List.of("402", "403", "404", "405", "425", "429", "436", "471") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $strain: List() from collect (Entity(type == "strain")) - then - componentCreationService.firstOrElse("Strain.0.0", "Strain", $strain, ""); - end - -rule "Conclusion.0.0: Unique values of Conclusion LD50" - when - $oecdNumber: String() from List.of("402", "403", "425", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $conclusions: List() from collect (Entity(type == "ld50_value")) - then - componentCreationService.joiningUnique("Conclusion.0.0", "Conclusion_LD50_mg_per_kg", $conclusions); - end - -rule "Conclusion0.1.0: Greater than found" - when - $oecdNumber: String() from List.of("402", "403", "425", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $conclusions: List(!isEmpty()) from collect (Entity(type == "ld50_greater")) - then - componentCreationService.create("Conclusion.1.0", "Conclusion_LD50_Greater_than", "Greater than", "Entity of type 'ld50_greater' found", $conclusions); - end - -rule "Conclusion.1.1: Greater than not found" - when - $oecdNumber: String() from List.of("402", "403", "425", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - not Entity(type == "ld50_greater") - then - componentCreationService.create("Conclusion.1.1", "Conclusion_LD50_Greater_than", "", "No entity of type 'ld50_greater' found"); - end - -rule "Conclusion.2.0: Minimum confidence as unique values" - when - $oecdNumber: String() from List.of("402", "403", "425", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $conclusions: List() from collect (Entity(type == "confidence_minimal")) - then - componentCreationService.joiningUnique("Conclusion.2.0", "Conclusion_Minimum_Confidence", $conclusions); - end - -rule "Conclusion.3.0: Maximum confidence as unique values" - when - $oecdNumber: String() from List.of("402", "403", "425", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $conclusions: List() from collect (Entity(type == "confidence_maximal")) - then - componentCreationService.joiningUnique("Conclusion.3.0", "Conclusion_Maximum_Confidence", $conclusions); - end - -rule "Necropsy.0.0: Necropsy findings from longest section" - when - FileAttribute(label == "OECD Number", value == "402") - $necropsies: List() from collect (Entity(type == "necropsy_findings")) - then - componentCreationService.joiningFromLongestSectionOnly("Necropsy.0.0", "Necropsy_Findings", $necropsies, " "); - end - -rule "Necropsy.0.1: Necropsy findings joined with \n" - when - FileAttribute(label == "OECD Number", value == "403" || value == "436") - $necropsies: List() from collect (Entity(type == "necropsy_findings")) - then - componentCreationService.joining("Necropsy.0.0", "Necropsy_Findings", $necropsies, "\n"); - end - -rule "Necropsy.1.0: Doses mg per kg of Bodyweight as one block" - when - FileAttribute(label == "OECD Number", value == "402") - $dosages: List() from collect (Entity(type == "doses_(mg_kg_bw)")) - then - componentCreationService.joining("Necropsy.1.0", "Doses_mg_per_kg_bw", $dosages, " "); - end - -rule "Necropsy.2.0: Conducted with 4 hours of exposure as one block" - when - $oecdNumber: String() from List.of("403", "436") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $exposures: List() from collect (Entity(type == "4h_exposure")) - then - componentCreationService.joining("Necropsy.3.0", "Conducted_with_4_Hours_of_Exposure", $exposures, " "); - end - -rule "StudyDesign.0.0: Study design as one block" - when - $oecdNumber: String() from List.of("404", "405", "429", "406", "428", "438", "439", "474", "487") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $studyDesigns: List() from collect (Entity(type == "study_design")) - then - componentCreationService.joining("StudyDesign.0.0", "Study_Design", $studyDesigns, " "); - end - -rule "Results.0.0: Results and conclusions as joined values" - when - $oecdNumber: String() from List.of("406", "428", "438", "439", "474", "487") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List() from collect (Entity(type == "results_and_conclusion")) - then - componentCreationService.joining("Results.0.0", "Results_and_Conclusions", $results, " "); - end - -rule "WeightBehavior.0.0: Weight change behavior as sentences" - when - FileAttribute(label == "OECD Number", value == "402") - $weightChanges: List() from collect (Entity(type == "weight_behavior_changes")) - then - componentCreationService.joining("WeightBehavior.0.0", "Weight_Behavior_Changes", $weightChanges, "\n"); - end - -rule "MortalityStatement.0.0: Mortality statements as one block" - when - FileAttribute(label == "OECD Number", value == "402") - $mortalityStatements: List() from collect (Entity(type == "mortality_statement")) - then - componentCreationService.joining("MortalityStatement.0.0", "Mortality_Statement", $mortalityStatements, " "); - end - -rule "ClinicalObservations.0.0: Clinical observations as sentences" - when - FileAttribute(label == "OECD Number", value == "403") - $observations: List() from collect (Entity(type == "clinical_observations")) - then - componentCreationService.joining("MortalityStatement.0.0", "Clinical_Observations", $observations, "\n"); - end - -rule "BodyWeight.0.0: Bodyweight changes as sentences" - when - FileAttribute(label == "OECD Number", value == "403") - $weightChanges: List() from collect (Entity(type == "bodyweight_changes")) - then - componentCreationService.joining("BodyWeight.0.0", "Body_Weight_Changes", $weightChanges, "\n"); - end - -rule "Detailing.0.0: Detailing of reported changes as one block" - when - $oecdNumber: String() from List.of("404", "405") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $detailings: List() from collect (Entity(type == "detailing")) - then - componentCreationService.joining("Detailing.0.0", "Detailing_of_Reported_Changes", $detailings, " "); - end - -rule "Sex.0.0: Male sex found" - when - $oecdNumber: String() from List.of("405", "429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $males: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "male" || value.toLowerCase() == "males"))) - then - componentCreationService.create("Sex.0.0", "Sex", "male", "male sex found", $males); - end - -rule "Sex.1.0: Female sex found" - when - $oecdNumber: String() from List.of("405", "429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $females: List(!isEmpty) from collect (Entity(type == "sex", (value.toLowerCase() == "female" || value.toLowerCase() == "females"))) - then - componentCreationService.create("Sex.0.0", "Sex", "female", "female sex found", $females); - end - -rule "NumberOfAnimals.0.0: Number of animals found" - when - $oecdNumber: String() from List.of("405", "429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $numberOfAnimals: Entity(type == "number_of_animals") - then - componentCreationService.create("NumberOfAnimals.0.0", "Number_of_Animals", $numberOfAnimals.getValue(), "Number of animals found directly", $numberOfAnimals); - end - -rule "NumberOfAnimals.1.0: Count unique occurences of animals" - when - $oecdNumber: String() from List.of("405", "429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - not Entity(type == "number_of_animals") - $animals: List() from collect (Entity(type == "animal_number")) - then - componentCreationService.uniqueValueCount("NumberOfAnimals.1.0", "Number_of_Animals", $animals); - end - -rule "ClinicalSigns.0.0: Clinical signs as sentences" - when - $oecdNumber: String() from List.of("425") - FileAttribute(label == "OECD Number", value == $oecdNumber) $clinicalSigns: List() from collect (Entity(type == "clinical_signs")) then - componentCreationService.joining("ClinicalSigns.0.0", "Clinical_Signs", $clinicalSigns, "\n"); + componentCreationService.joiningFromSameTableRow("IndivRowExtr.1.2", "1.2 Individual Rows", $clinicalSigns); end -rule "DoseMortality.0.0: Dose mortality joined with dose from same table row" + +rule "IndivColExtr.1.3: Individual column of table" + when + $tableColValues: List() from collect (Entity(type == "dosages")) + then + componentCreationService.joiningFromSameTableRow("IndivColExtr.1.3", "1.3 Individual Column", $tableColValues); + end + + +rule "CombColExtr.2.1: Combined Columns Extraction" when - $oecdNumber: String() from List.of("425") - FileAttribute(label == "OECD Number", value == $oecdNumber) $doseMortalities: List() from collect (Entity(type == "dose_mortality" || type == "dose_mortality_dose")) then - componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "Dose_Mortality", $doseMortalities); + componentCreationService.joiningFromSameTableRow("DoseMortality.0.0", "2.1 Combined Columns", $doseMortalities); end -rule "Mortality.0.0: Mortality as one block" + +rule "ValueExtrRef.2.2: Cells containing dose for survived males" when - $oecdNumber: String() from List.of("425") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $mortalities: List() from collect (Entity(type == "mortality")) + $tableValues: List() from collect (Entity(type == "doses_mg_kg_bw")) then - componentCreationService.joining("Mortality.0.0", "Mortality", $mortalities, " "); + componentCreationService.joining("ValueExtrRef.2.2", "2.2 Individual Table Values", $tableValues); end -rule "Dosages.0.0: First found value of Dosages" + +rule "AdvTableExtr.2.3: Cells containing dose for survived males" when - $oecdNumber: String() from List.of("425") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $mortalities: List() from collect (Entity(type == "dosages")) + $tableValues: List() from collect (Entity(type == "experiment_female_survived")) then - componentCreationService.firstOrElse("Dosages.0.0", "Dosages", $mortalities, ""); + componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues); end -rule "PrelimResults.0.0: Preliminary test results as sentences" - when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List() from collect (Entity(type == "preliminary_test_results")) - then - componentCreationService.joining("PrelimResults.0.0", "Preliminary_Test_Results", $results, "\n"); - end -rule "TestResults.0.0: Test results as one block" +rule "EntityBasedExtr.2.4: Cells containing dose for survived males" when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List() from collect (Entity(type == "test_results")) + $tableValues: List() from collect (Entity(type == "study_design")) then - componentCreationService.joining("TestResults.0.0", "Test_Results", $results, " "); + componentCreationService.joiningFromSameTableRow("EntityBasedExtr.2.4", "2.4 Entity-Based Values", $tableValues); end - -rule "PositiveControl.0.0: Was the definitive study conducted with positive control" - when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List() from collect (Entity(type == "positive_control")) - then - componentCreationService.joining("PositiveControl.0.0", "Was_the_definitive_study_conducted_with_positive_control", $results, " "); - end - -rule "MainResults.0.0: Results from main study as one block" - when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List() from collect (Entity(type == "results_(main_study)")) - then - componentCreationService.joining("MainResults.0.0", "Results_Main_Study", $results, " "); - end - -rule "UsedApproach.0.0: Used approach found and mapped to 'Group'" - when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - $results: List(!isEmpty()) from collect (Entity(type == "approach_used")) - then - componentCreationService.create("UsedApproach.0.0", "What_was_the_approach_used", "Group", "'Group' when approach used is present, else 'Individual'", $results); - end - -rule "UsedApproach.1.0: Used approach not found and thus 'Individual'" - when - $oecdNumber: String() from List.of("429") - FileAttribute(label == "OECD Number", value == $oecdNumber) - not Entity(type == "approach_used") - then - componentCreationService.create("UsedApproach.1.0", "What_was_the_approach_used", "Individual", "'Group' when approach used is present, else 'Individual'"); - end - -/* -rule "DefaultComponents.999.0: Create components for all unmapped entities." - salience -999 - when - $allEntities: List(!isEmpty()) from collect (Entity()) - then - componentCreationService.createComponentsForUnMappedEntities("DefaultComponents.999.0", $allEntities); - end -*/ - -//------------------------------------ Component merging rules ------------------------------------ -/* -rule "X.0.0: merge duplicate component references" - when - $first: Component() - $duplicate: Component(this != $first, name == $first.name, value == $first.value) - then - $first.getReferences().addAll($duplicate.getReferences()); - retract($duplicate); - end -*/ \ No newline at end of file diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl new file mode 100644 index 00000000..72b65851 --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_components.drl @@ -0,0 +1,55 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.component.Component; +import com.iqser.red.service.redaction.v1.server.model.component.Entity; +import com.iqser.red.service.redaction.v1.server.service.document.ComponentCreationService; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Change; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryState; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.EntryType; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.ManualChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Position; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; + +global ComponentCreationService componentCreationService + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +query "getComponents" + $component: Component() + end + +//------------------------------------ Guideline mapping object ------------------------------------ + +declare GuidelineMapping + number: String + year: String + guideline: String + end + +//------------------------------------ Test Components rules ------------------------------------ + +rule "AdvTableExtr.2.3: Cells containing dose for survived males" + when + $tableValues: List() from collect (Entity(type == "test")) + then + componentCreationService.rowValueCount("AdvTableExtr.2.3", "2.3 Advanced Table Values", $tableValues); + end diff --git a/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl new file mode 100644 index 00000000..ccf3005a --- /dev/null +++ b/redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl @@ -0,0 +1,401 @@ +package drools + +import static java.lang.String.format; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch; +import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch; + +import java.util.List; +import java.util.LinkedList; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.Collection; +import java.util.stream.Stream; +import java.util.Optional; + +import com.iqser.red.service.redaction.v1.server.model.document.*; +import com.iqser.red.service.redaction.v1.server.model.document.TextRange; +import com.iqser.red.service.redaction.v1.server.model.document.entity.*; +import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType; +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule; +import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity +import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule +import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header; +import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.*; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock; +import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock; +import com.iqser.red.service.redaction.v1.server.model.NerEntities; +import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary; +import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel; +import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService; +import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService; +import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility; + +import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute; +import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange; +import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.AnnotationStatus; + +global Document document +global EntityCreationService entityCreationService +global ManualChangesApplicationService manualChangesApplicationService +global Dictionary dictionary + +//------------------------------------ queries ------------------------------------ + +query "getFileAttributes" + $fileAttribute: FileAttribute() + end + +//------------------------------------ Local dictionary search rules ------------------------------------ + +// Rule unit: LocalDictionarySearch.0 +rule "LDS.0.0: Run local dictionary search" + agenda-group "LOCAL_DICTIONARY_ADDS" + salience -999 + when + $dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels() + then + entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document) + .forEach(entity -> { + Collection matchedRules = $dictionaryModel.getLocalEntriesWithMatchedRules().get(entity.getValue()); + entity.addMatchedRules(matchedRules); + }); + end +// --------------------------------------- Your rules below this line -------------------------------------------------- + +rule "TAB.0.0: Study Type File Attribute" + when + not FileAttribute(label == "OECD Number", valueEqualsAnyOf("402","403","404","405","425","429","436","438","439","471","487")) + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") + && containsAnyString("OECD", "EPA", "OPPTS")) + then + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|(?:.{5,40}(?:Number |Procedure |Guideline )))(4[\\d]{2})", 1 ,$section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + RedactionSearchUtility.findTextRangesByRegexIgnoreCase("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", 1, $section.getTextBlock()).stream() + .map(boundary -> $section.getTextBlock().subSequence(boundary).toString()) + .map(value -> FileAttribute.builder().label("OECD Number").value(value).build()) + .forEach(fileAttribute -> insert(fileAttribute)); + end + +rule "TAB.0.1: Guidelines" + when + $section: Section(containsAnyString("DATA REQUIREMENT", "TEST GUIDELINE", "MÉTODO(S) DE REFERÊNCIA(S):") && containsAnyString("OECD", "EPA", "OPPTS")) + then + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2})", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline no. found")); + entityCreationService.byRegex("(?<=OECD)(?:[\\w\\s,\\[\\]\\(\\)\\.]{1,10}|.{5,40}(?:Number |Procedure |Guideline ))(4[\\d]{2}),?\\s\\(?(\\d{4})\\)?", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD)[\\w\\s,\\[\\]]{1,10}\\((\\d{4})\\)\\s(4[\\d]{2})", "oecd_guideline_year", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_number", EntityType.ENTITY, 1, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline number found")); + entityCreationService.byRegex("(?<=OECD).{5,40}Method (4[\\d]{2}).{1,65}(\\d{4})\\)", "oecd_guideline_year", EntityType.ENTITY, 2, $section) + .forEach(guideline -> guideline.apply("TAB.0.1", "OECD Guideline year found")); + end + + +rule "TAB.6.0: Targeted cell extraction (Experimental Stop date)" + when + $section: Section(containsString("Maximum occurrence")) + $table: Table() from $section.streamChildren().toList() + TableCell(containsWordIgnoreCase("water"), $row: row) from $table.streamTableCells().toList() + $test: TableCell($row == row) from $table.streamTableCells().toList() + then + System.out.println("AAAA: " + $test); + entityCreationService.bySemanticNode($test, "test", EntityType.ENTITY) + .ifPresent(entity -> entity.apply("TAB.6.0", "Some test stuff")); + end + + +//------------------------------------ Manual redaction rules ------------------------------------ + +// Rule unit: MAN.0 +rule "MAN.0.0: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $entityToBeResized: TextEntity(matchesAnnotationId($id)) + then + manualChangesApplicationService.resizeEntityAndReinsert($entityToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($entityToBeResized); + $entityToBeResized.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.0.1: Apply manual resize redaction" + salience 128 + when + $resizeRedaction: ManualResizeRedaction($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate)) + $imageToBeResized: Image(id == $id) + then + manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction); + retract($resizeRedaction); + update($imageToBeResized); + update($imageToBeResized.getParent()); + end + + +// Rule unit: MAN.1 +rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeRemoved: TextEntity(matchesAnnotationId($id)) + then + $entityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($entityToBeRemoved); + retract($idRemoval); + $entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node)); + end + +rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image" + salience 128 + when + $idRemoval: IdRemoval($id: annotationId, status == AnnotationStatus.APPROVED) + $imageEntityToBeRemoved: Image($id == id) + then + $imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval); + update($imageEntityToBeRemoved); + retract($idRemoval); + update($imageEntityToBeRemoved.getParent()); + end + + +// Rule unit: MAN.2 +rule "MAN.2.0: Apply force redaction" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToForce: TextEntity(matchesAnnotationId($id)) + then + $entityToForce.getManualOverwrite().addChange($force); + update($entityToForce); + $entityToForce.getIntersectingNodes().forEach(node -> update(node)); + retract($force); + end + +rule "MAN.2.1: Apply force redaction to images" + salience 128 + when + $force: ManualForceRedaction($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToForce: Image(id == $id) + then + $imageToForce.getManualOverwrite().addChange($force); + update($imageToForce); + update($imageToForce.getParent()); + retract($force); + end + + +// Rule unit: MAN.3 +rule "MAN.3.0: Apply entity recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type != $type) + then + $entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node)); + manualChangesApplicationService.recategorize($entityToBeRecategorized, $recategorization); + retract($recategorization); + // Entity is copied and inserted, so the old entity needs to be retracted to avoid duplication. + retract($entityToBeRecategorized); + end + +rule "MAN.3.1: Apply entity recategorization of same type" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, $type: type, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type == $type) + then + $entityToBeRecategorized.getManualOverwrite().addChange($recategorization); + retract($recategorization); + end + +rule "MAN.3.2: Apply image recategorization" + salience 128 + when + $recategorization: ManualRecategorization($id: annotationId, status == AnnotationStatus.APPROVED, $requestDate: requestDate) + not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate)) + $imageToBeRecategorized: Image($id == id) + then + manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization); + update($imageToBeRecategorized); + update($imageToBeRecategorized.getParent()); + retract($recategorization); + end + + +// Rule unit: MAN.4 +rule "MAN.4.0: Apply legal basis change" + salience 128 + when + $legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $imageToBeRecategorized: Image($id == id) + then + $imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); + end + +rule "MAN.4.1: Apply legal basis change" + salience 128 + when + $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) + $entityToBeChanged: TextEntity(matchesAnnotationId($id)) + then + $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); + end + + +//------------------------------------ Entity merging rules ------------------------------------ + +// Rule unit: X.0 +rule "X.0.0: Remove Entity contained by Entity of same type" + salience 65 + when + $larger: TextEntity($type: type, $entityType: entityType, active()) + $contained: TextEntity(containedBy($larger), type == $type, entityType == $entityType, this != $larger, !hasManualChanges(), active()) + then + $contained.remove("X.0.0", "remove Entity contained by Entity of same type"); + retract($contained); + end + + +// Rule unit: X.1 +rule "X.1.0: Merge intersecting Entities of same type" + salience 64 + when + $first: TextEntity($type: type, $entityType: entityType, !resized(), active()) + $second: TextEntity(intersects($first), type == $type, entityType == $entityType, this != $first, !hasManualChanges(), active()) + then + TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document); + $first.remove("X.1.0", "merge intersecting Entities of same type"); + $second.remove("X.1.0", "merge intersecting Entities of same type"); + retract($first); + retract($second); + mergedEntity.getIntersectingNodes().forEach(node -> update(node)); + end + + +// Rule unit: X.2 +rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE" + salience 64 + when + $falsePositive: TextEntity($type: type, entityType == EntityType.FALSE_POSITIVE, active()) + $entity: TextEntity(containedBy($falsePositive), type == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges(), active()) + then + $entity.getIntersectingNodes().forEach(node -> update(node)); + $entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE"); + retract($entity) + end + + +// Rule unit: X.3 +rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION" + salience 64 + when + $falseRecommendation: TextEntity($type: type, entityType == EntityType.FALSE_RECOMMENDATION, active()) + $recommendation: TextEntity(containedBy($falseRecommendation), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"); + retract($recommendation); + end + + +// Rule unit: X.4 +rule "X.4.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY with same type" + salience 256 + when + $entity: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(intersects($entity), type == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $entity.addEngines($recommendation.getEngines()); + $recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY with same type"); + retract($recommendation); + end + + +// Rule unit: X.5 +rule "X.5.0: Remove Entity of type RECOMMENDATION when contained by ENTITY" + salience 256 + when + $entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $recommendation: TextEntity(containedBy($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges(), active()) + then + $recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when contained by ENTITY"); + retract($recommendation); + end + + +// Rule unit: X.6 +rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type ENTITY" + salience 32 + when + $higherRank: TextEntity($type: type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(containedBy($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active()) + then + $lowerRank.getIntersectingNodes().forEach(node -> update(node)); + $lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY"); + retract($lowerRank); + end + + +rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" + salience 32 + when + $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) + $lowerRank: TextEntity(intersects($higherRank), type != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) + then + $higherRank.getIntersectingNodes().forEach(node -> update(node)); + $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); + retract($higherRank); + end + + +// Rule unit: X.7 +rule "X.7.0: remove all images" + salience 512 + when + $image: Image(imageType != ImageType.OCR, !hasManualChanges()) + then + $image.remove("X.7.0", "remove all images"); + retract($image); + end + + +//------------------------------------ File attributes rules ------------------------------------ + +// Rule unit: FA.1 +rule "FA.1.0: Remove duplicate FileAttributes" + salience 64 + when + $fileAttribute: FileAttribute($label: label, $value: value) + $duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value) + then + retract($duplicate); + end +