Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-10471' into 'release/4.423.x'

Browse Source 
RED-10471: PII.11.0 does not redact anymore

See merge request redactmanager/redaction-service!555
This commit is contained in:
Maverick Studer 2024-11-14 18:24:40 +01:00
commit 99f2c1f4f2
20 changed files with 1174 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1" description = "redaction-service-server-v1"
val layoutParserVersion = "0.181.0" val layoutParserVersion = "0.191.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"

15
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -55,14 +55,13 @@ global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger global RulesLogger logger
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"
$fileAttribute: FileAttribute() $fileAttribute: FileAttribute()
end end
//------------------------------------ H rules ------------------------------------ //------------------------------------ Headlines rules ------------------------------------
// Rule unit: H.0 // Rule unit: H.0
rule "H.0.0: retract table of contents page" rule "H.0.0: retract table of contents page"
@ -129,6 +128,7 @@ rule "H.3.1: Study Type File Attribute in Headlines"
.ifPresent(fileAttribute -> insert(fileAttribute)); .ifPresent(fileAttribute -> insert(fileAttribute));
end end
//------------------------------------ General documine rules ------------------------------------ //------------------------------------ General documine rules ------------------------------------
// Rule unit: DOC.1 // Rule unit: DOC.1
@ -296,6 +296,7 @@ rule "DOC.1.4: Guideline in Headlines"
); );
end end
// Rule unit: DOC.2 // Rule unit: DOC.2
rule "DOC.2.0: Report number" rule "DOC.2.0: Report number"
when when
@ -1147,6 +1148,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
.forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a")); .forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
end end
//------------------------------------ Table extraction rules ------------------------------------ //------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0 // Rule unit: TAB.0
@ -1296,7 +1298,8 @@ rule "TAB.7.0: Indicator (Species)"
.ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found")); .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
end end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0 // Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.0: Apply manual resize redaction"
@ -1424,6 +1427,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
@ -1485,7 +1489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1534,8 +1537,6 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
retract($recommendation); retract($recommendation);
end end
// Rule unit: X.5
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256 salience 256
when when
@ -1580,6 +1581,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
retract($other); retract($other);
end end
// Rule unit: X.9 // Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures" rule "X.9.0: Merge mostly contained signatures"
when when
@ -1590,6 +1592,7 @@ rule "X.9.0: Merge mostly contained signatures"
$signature.addEngine(LayoutEngine.AI); $signature.addEngine(LayoutEngine.AI);
end end
// Rule unit: X.10 // Rule unit: X.10
rule "X.10.0: remove false positives of ai" rule "X.10.0: remove false positives of ai"
when when

5
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl
View File

@ -34,11 +34,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
/**
The imports, globals, queries and rules from this file are required for any component rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"

8
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl
View File

@ -61,12 +61,6 @@ global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
/**
The imports, globals, queries and rules from this file are required for any entity rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"
@ -75,7 +69,7 @@ query "getFileAttributes"
//------------------------------------ Local dictionary search rules ------------------------------------ //------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0 // Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search" rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999 salience -999

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -14,6 +14,7 @@ import java.util.UUID;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -326,6 +327,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Test @Test
@SneakyThrows @SneakyThrows
@Disabled
void testNerEntitiesAfterReanalysis() { void testNerEntitiesAfterReanalysis() {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -15,6 +15,7 @@ import java.util.List;
import org.apache.pdfbox.Loader; import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -134,6 +135,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
@Test @Test
@SneakyThrows @SneakyThrows
@Disabled
public void testBuildTextBlockPerformance() { public void testBuildTextBlockPerformance() {
int n = 10000; int n = 10000;

15
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service.document; package com.iqser.red.service.redaction.v1.server.service.document;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
@ -126,7 +127,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntity.getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
.get(0).x(), 355.53775f); .get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
@ -173,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -256,7 +257,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(resizedAoel.isPresent()); assertTrue(resizedAoel.isPresent());
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
assertEquals(resizedAoel.get().getTextBefore(), "is below the "); assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
assertEquals(resizedAoel.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(resizedAoel.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
@ -272,7 +273,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(cormsResized.isPresent()); assertTrue(cormsResized.isPresent());
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of"); assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore "); assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the"); assertThat(cormsResized.get().getSection()).contains("Paragraph: The Co-RMS indicated the");
assertEquals(cormsResized.get().getPositions() assertEquals(cormsResized.get().getPositions()
.get(0).x(), positions2.get(0).getTopLeftX()); .get(0).x(), positions2.get(0).getTopLeftX());
assertEquals(cormsResized.get().getPositions() assertEquals(cormsResized.get().getPositions()
@ -288,7 +289,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(a9Resized.isPresent()); assertTrue(a9Resized.isPresent());
assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
assertEquals(a9Resized.get().getTextBefore(), "data for S"); assertEquals(a9Resized.get().getTextBefore(), "data for S");
assertEquals(a9Resized.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(a9Resized.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
.get(0).x(), positions3.get(0).getTopLeftX()); .get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
@ -338,7 +339,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -388,7 +389,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: With respect to the");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -273,6 +274,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
$table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity)); $table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
end end
// Rule unit: CBI.16 // Rule unit: CBI.16
rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables" rule "CBI.16.0: Do not redact Names and Addresses if published information found in Section without tables"
when when
@ -874,7 +876,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1291,7 +1293,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1430,7 +1431,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

34
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -340,7 +341,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end end
// Rule unit: CBI.12 - table rules remains // Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
@ -404,8 +405,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator" rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -422,8 +421,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
}); });
end end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate" rule "CBI.12.5: Redacted because table row contains a vertebrate"
when when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -440,7 +437,6 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
}); });
end end
rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'" rule "CBI.12.6: Skip Addresses on TableCell with header 'Owner'"
salience -1 salience -1
when when
@ -535,7 +531,7 @@ rule "CBI.12.15: Redacted because table row contains a vertebrate, a no_redactio
end end
// Rule unit: CBI.13 - section rules // Rule unit: CBI.13
rule "CBI.13.0: Ignore CBI Address recommendations" rule "CBI.13.0: Ignore CBI Address recommendations"
when when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -545,7 +541,6 @@ rule "CBI.13.0: Ignore CBI Address recommendations"
retract($entity) retract($entity)
end end
// from CBI.3.0
rule "CBI.13.1: Redacted because Section contains a vertebrate" rule "CBI.13.1: Redacted because Section contains a vertebrate"
when when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -561,7 +556,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
}); });
end end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -570,8 +564,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found")); .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section" rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when when
$section: Section(!hasTables(), $section: Section(!hasTables(),
@ -589,8 +581,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
}); });
end end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section" rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when when
$section: Section(!hasTables(), $section: Section(!hasTables(),
@ -612,7 +602,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
}); });
end end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity" rule "CBI.13.5: Redacted because Section contains must_redact entity"
when when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -628,6 +617,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
}); });
end end
// Rule unit: CBI.14 // Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when when
@ -742,6 +732,7 @@ rule "CBI.16.3: Do not redact PII if published information found in same table r
$pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii)); $pii.skipWithReferences("CBI.16.3", "Published Information found in row", $table.getEntitiesOfTypeInSameRow("published_information", $pii));
end end
// Rule unit: CBI.17 // Rule unit: CBI.17
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon" rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
when when
@ -847,7 +838,6 @@ rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJEC
// Rule unit: CBI.21 // Rule unit: CBI.21
// from CBI.6
rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section" rule "CBI.21.0: Do not redact Names and Addresses if published_information is found in Section"
when when
$section: Section(!hasTables(), $section: Section(!hasTables(),
@ -882,7 +872,6 @@ rule "CBI.21.1: Do not redact Names and Addresses if published_information is fo
}); });
end end
rule "CBI.21.2: Redact short Authors section (non vertebrate study)" rule "CBI.21.2: Redact short Authors section (non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1397,7 +1386,6 @@ rule "PII.8.2: Redact contact information if producer is found (vertebrate study
end end
// UPDATED WITH LIMIT
// Rule unit: PII.9 // Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\"" rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
when when
@ -1444,7 +1432,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)" rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1458,7 +1445,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1475,8 +1462,6 @@ rule "PII.12.0: Expand PII entities with salutation prefix"
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end end
// Rule unit: PII.12
rule "PII.12.1: Expand PII entities with salutation prefix" rule "PII.12.1: Expand PII entities with salutation prefix"
when when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -1591,7 +1576,6 @@ rule "ETC.3.3: Redact logos"
$logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"); $logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
end end
// from preGFL Knoell
rule "ETC.3.4: Skip logos" rule "ETC.3.4: Skip logos"
when when
$logo: Image(imageType == ImageType.LOGO) $logo: Image(imageType == ImageType.LOGO)
@ -1759,6 +1743,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found"); $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end end
//------------------------------------ AI rules ------------------------------------ //------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0 // Rule unit: AI.0
@ -1885,7 +1870,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end end
//------------------------------------ Manual redaction rules ------------------------------------ //------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0 // Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.0: Apply manual resize redaction"
@ -2075,7 +2060,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -2214,7 +2198,6 @@ rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
retract($dictionaryEntity); retract($dictionaryEntity);
end end
rule "X.11.1: Remove non manual entity which intersects with a manual entity" rule "X.11.1: Remove non manual entity which intersects with a manual entity"
salience 64 salience 64
when when
@ -2225,7 +2208,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

1
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1419,7 +1419,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when

10
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -139,6 +140,7 @@ rule "CBI.0.4: Redact CBI Authors (vertebrate Study)"
$entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002"); $entity.redact("CBI.0.4", "Author found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
end end
// Rule unit: CBI.1 // Rule unit: CBI.1
rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)" rule "CBI.1.0: Do not redact CBI Address (non vertebrate Study)"
when when
@ -439,7 +441,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -599,7 +600,6 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end end
rule "PII.10.1: Redact study director abbreviation (vertebrate study)" rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
when when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -613,7 +613,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -708,6 +708,7 @@ rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confi
retract($dossierRedaction); retract($dossierRedaction);
end end
// Rule unit: ETC.12 // Rule unit: ETC.12
rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)" rule "ETC.12.2: Skip dossier_redaction (Non vertebrate study)"
when when
@ -725,6 +726,7 @@ rule "ETC.12.3: Skip dossier_redaction (Vertebrate study)"
$dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found"); $dossierRedaction.skip("ETC.12.3", "Dossier dictionary entry found");
end end
//------------------------------------ AI rules ------------------------------------ //------------------------------------ AI rules ------------------------------------
// Rule unit: AI.0 // Rule unit: AI.0
@ -1018,7 +1020,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1157,7 +1158,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

4
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -149,7 +149,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end end
//------------------------------------ Manual redaction rules ------------------------------------ //------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0 // Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.0: Apply manual resize redaction"
@ -339,7 +339,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -478,7 +477,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

24
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -85,6 +86,7 @@ rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
//------------------------------------ CBI rules ------------------------------------ //------------------------------------ CBI rules ------------------------------------
// Rule unit: CBI.0 // Rule unit: CBI.0
rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx" rule "CBI.0.0: Add CBI_author with \"et al.\" RegEx"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
@ -177,7 +179,7 @@ rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study
end end
// Rule unit: CBI.12 - table rules remains // Rule unit: CBI.12
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'" rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
when when
@ -241,8 +243,6 @@ rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'V
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study")); .ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
end end
//from CBI.3.3
rule "CBI.12.4: Redacted because table row contains a redaction_indicator" rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
when when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -259,8 +259,6 @@ rule "CBI.12.4: Redacted because table row contains a redaction_indicator"
}); });
end end
//from CBI.3.1
rule "CBI.12.5: Redacted because table row contains a vertebrate" rule "CBI.12.5: Redacted because table row contains a vertebrate"
when when
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")) $table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
@ -278,8 +276,7 @@ rule "CBI.12.5: Redacted because table row contains a vertebrate"
end end
// Rule unit: CBI.13 - section rules // Rule unit: CBI.13
// from CBI.3.0
rule "CBI.13.1: Redacted because Section contains a vertebrate" rule "CBI.13.1: Redacted because Section contains a vertebrate"
when when
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -295,7 +292,6 @@ rule "CBI.13.1: Redacted because Section contains a vertebrate"
}); });
end end
//from CBI.3.2
rule "CBI.13.2: Do not redact because Section does not contain a vertebrate" rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
when when
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -304,8 +300,6 @@ rule "CBI.13.2: Do not redact because Section does not contain a vertebrate"
.forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found")); .forEach(entity -> entity.skip("CBI.13.2", "No vertebrate found"));
end end
// from CBI.4.0
rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section" rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction_indicator is found in Section"
when when
$section: Section(!hasTables(), $section: Section(!hasTables(),
@ -323,8 +317,6 @@ rule "CBI.13.3: Do not redact Names and Addresses if vertebrate and no_redaction
}); });
end end
// from CBI.5.0
rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section" rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indicator but also redaction_indicator is found in Section"
when when
$section: Section(!hasTables(), $section: Section(!hasTables(),
@ -346,7 +338,6 @@ rule "CBI.13.4: Redact Names and Addresses if vertebrate and no_redaction_indica
}); });
end end
// From CBI.8.0
rule "CBI.13.5: Redacted because Section contains must_redact entity" rule "CBI.13.5: Redacted because Section contains must_redact entity"
when when
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))) $section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
@ -362,6 +353,7 @@ rule "CBI.13.5: Redacted because Section contains must_redact entity"
}); });
end end
// Rule unit: CBI.14 // Rule unit: CBI.14
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\"" rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
when when
@ -854,7 +846,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -1133,7 +1125,7 @@ rule "AI.7.0: Add all NER Entities of type Address"
end end
//------------------------------------ Manual redaction rules ------------------------------------ //------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0 // Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.0: Apply manual resize redaction"
@ -1323,7 +1315,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1462,7 +1453,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -397,7 +397,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -536,7 +535,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

2
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -489,7 +489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -639,7 +638,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

23
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -389,6 +389,15 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
@ -401,19 +410,6 @@ rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
// Rule unit: X.2
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -563,7 +559,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

1108
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json
View File

File diff suppressed because one or more lines are too long

8
redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl
View File

@ -4,7 +4,13 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
global RulesLogger logger global RulesLogger logger
//------------------------------------ queries ------------------------------------
//------------------------------------ LOG rules ------------------------------------
// Rule unit: LOG.0
rule "LOG.0.0: Test log info" rule "LOG.0.0: Test log info"
salience 1 salience 1
when when
@ -27,4 +33,4 @@ rule "LOG.0.2: Test log error"
then then
String result = null; String result = null;
result.toString(); result.toString();
end end

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -420,7 +421,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -593,7 +593,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -901,7 +901,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1040,7 +1039,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

3
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -1459,7 +1460,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));