RED-10471: PII.11.0 does not redact anymore

This commit is contained in:
Maverick Studer 2024-11-14 18:24:40 +01:00
parent de76cb96f9
commit 99740041be
20 changed files with 1174 additions and 106 deletions

View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1"
val layoutParserVersion = "0.181.0"
val layoutParserVersion = "0.191.0"
val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0"

View File

@ -55,14 +55,13 @@ global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
global RulesLogger logger
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
$fileAttribute: FileAttribute()
end
//------------------------------------ H rules ------------------------------------
//------------------------------------ Headlines rules ------------------------------------
// Rule unit: H.0
rule "H.0.0: retract table of contents page"
@ -129,6 +128,7 @@ rule "H.3.1: Study Type File Attribute in Headlines"
.ifPresent(fileAttribute -> insert(fileAttribute));
end
//------------------------------------ General documine rules ------------------------------------
// Rule unit: DOC.1
@ -296,6 +296,7 @@ rule "DOC.1.4: Guideline in Headlines"
);
end
// Rule unit: DOC.2
rule "DOC.2.0: Report number"
when
@ -1147,6 +1148,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
.forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
end
//------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0
@ -1296,7 +1298,8 @@ rule "TAB.7.0: Indicator (Species)"
.ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1424,6 +1427,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
@ -1485,7 +1489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1534,8 +1537,6 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
retract($recommendation);
end
// Rule unit: X.5
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256
when
@ -1580,6 +1581,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
retract($other);
end
// Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures"
when
@ -1590,6 +1592,7 @@ rule "X.9.0: Merge mostly contained signatures"
$signature.addEngine(LayoutEngine.AI);
end
// Rule unit: X.10
rule "X.10.0: remove false positives of ai"
when

View File

@ -34,11 +34,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
global ComponentCreationService componentCreationService
/**
The imports, globals, queries and rules from this file are required for any component rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------
query "getFileAttributes"

View File

@ -61,12 +61,6 @@ global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary
/**
The imports, globals, queries and rules from this file are required for any entity rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------
query "getFileAttributes"
@ -75,7 +69,7 @@ query "getFileAttributes"
//------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0
// Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999

View File

@ -14,6 +14,7 @@ import java.util.UUID;
import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
@ -326,6 +327,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Test
@SneakyThrows
@Disabled
void testNerEntitiesAfterReanalysis() {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");

View File

@ -15,6 +15,7 @@ import java.util.List;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired;
@ -134,6 +135,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
@Test
@SneakyThrows
@Disabled
public void testBuildTextBlockPerformance() {
int n = 10000;

View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service.document;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ -126,7 +127,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertThat(unprocessedManualEntity.getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getPositions()
.get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions()
@ -173,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -256,7 +257,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(resizedAoel.isPresent());
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
assertEquals(resizedAoel.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertThat(resizedAoel.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(resizedAoel.get().getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions()
@ -272,7 +273,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(cormsResized.isPresent());
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the");
assertThat(cormsResized.get().getSection()).contains("Paragraph: The Co-RMS indicated the");
assertEquals(cormsResized.get().getPositions()
.get(0).x(), positions2.get(0).getTopLeftX());
assertEquals(cormsResized.get().getPositions()
@ -288,7 +289,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(a9Resized.isPresent());
assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
assertEquals(a9Resized.get().getTextBefore(), "data for S");
assertEquals(a9Resized.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertThat(a9Resized.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(a9Resized.get().getPositions()
.get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions()
@ -338,7 +339,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L");
assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -388,7 +389,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the");
assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: With respect to the");
assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions()

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -273,6 +274,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
$table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end
// Rule unit: CBI.16
rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
when
@ -874,7 +876,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1291,7 +1293,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1430,7 +1431,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -340,7 +341,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end
// Rule unit: CBI.12 - table rules remains
// Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
@ -404,8 +405,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -422,8 +421,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
});
end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -440,7 +437,6 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
});
end
rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'"
salience -1
when
@ -535,7 +531,7 @@ rule "CBI.12.15: Redacted because table row contains a vertebrate, a no_redactio
end
// Rule unit: CBI.13 - section rules
// Rule unit: CBI.13
rule "CBI.13.0: Ignore CBI Address recommendations"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -545,7 +541,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
retract($entity)
end
// from CBI.3.0
rule "CBI.13.1: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -561,7 +556,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
});
end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -570,8 +564,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -589,8 +581,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
});
end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -612,7 +602,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
});
end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -628,6 +617,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
});
end
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
@ -742,6 +732,7 @@ rule "CBI.16.3: Do not redact PII if published information found in same table r
$pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
end
// Rule unit: CBI.17
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
when
@ -847,7 +838,6 @@ rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
// Rule unit: CBI.21
// from CBI.6
rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section"
when
$section: Section(!hasTables(),
@ -882,7 +872,6 @@ rule "CBI.21.1: Do not redact Names and Addresses if published_information is fo
});
end
rule "CBI.21.2: Redact short Authors section (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1397,7 +1386,6 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
end
// UPDATED WITH LIMIT
// Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when
@ -1444,7 +1432,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1458,7 +1445,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1475,8 +1462,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.12
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1591,7 +1576,6 @@ rule "ETC.3.3: Redact logos"
$logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
end
// from preGFL Knoell
rule "ETC.3.4: Skip logos"
when
$logo: Image(imageType == ImageType.LOGO)
@ -1759,6 +1743,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1885,7 +1870,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -2075,7 +2060,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -2214,7 +2198,6 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
retract($dictionaryEntity);
end
rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64
when
@ -2225,7 +2208,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -1419,7 +1419,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -139,6 +140,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end
// Rule unit: CBI.1
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when
@ -439,7 +441,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -599,7 +600,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -613,7 +613,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -708,6 +708,7 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
retract($dossierRedaction);
end
// Rule unit: ETC.12
rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
when
@ -725,6 +726,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end
//------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0
@ -1018,7 +1020,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1157,7 +1158,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -339,7 +339,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -478,7 +477,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -85,6 +86,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
//------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS"
@ -177,7 +179,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end
// Rule unit: CBI.12 - table rules remains
// Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS"
when
@ -241,8 +243,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -259,8 +259,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
});
end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate"
when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -278,8 +276,7 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
end
// Rule unit: CBI.13 - section rules
// from CBI.3.0
// Rule unit: CBI.13
rule "CBI.13.1: Redacted because Section contains a vertebrate"
when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -295,7 +292,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
});
end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -304,8 +300,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -323,8 +317,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
});
end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when
$section: Section(!hasTables(),
@ -346,7 +338,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
});
end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity"
when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -362,6 +353,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
});
end
// Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when
@ -854,7 +846,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1133,7 +1125,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction"
@ -1323,7 +1315,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1462,7 +1453,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -397,7 +397,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -536,7 +535,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -489,7 +489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -639,7 +638,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -389,6 +389,15 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
@ -401,19 +410,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
retract($entity)
end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -563,7 +559,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -4,7 +4,13 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
global RulesLogger logger
//------------------------------------ queries ------------------------------------
//------------------------------------ LOG rules ------------------------------------
// Rule unit: LOG.0
rule "LOG.0.0: Test log info"
salience 1
when
@ -27,4 +33,4 @@ rule "LOG.0.2: Test log error"
then
String result = null;
result.toString();
end
end

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -420,7 +421,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -593,7 +593,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -901,7 +901,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
@ -1040,7 +1039,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity);
end
rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70
when

View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -1459,7 +1460,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
$section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));