Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

Merge branch 'RED-10471-fp' into 'master'

Browse Source 
RED-10471: PII.11.0 does not redact anymore

Closes RED-10471

See merge request redactmanager/redaction-service!556
This commit is contained in:
Maverick Studer 2024-11-14 18:41:51 +01:00
commit 681d6328ef
16 changed files with 1170 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -12,7 +12,7 @@ plugins {
description = "redaction-service-server-v1" description = "redaction-service-server-v1"
val layoutParserVersion = "0.181.0" val layoutParserVersion = "0.191.0"
val jacksonVersion = "2.15.2" val jacksonVersion = "2.15.2"
val droolsVersion = "9.44.0.Final" val droolsVersion = "9.44.0.Final"
val pdfBoxVersion = "3.0.0" val pdfBoxVersion = "3.0.0"

15
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/all_rules_documine.drl
View File

@ -55,14 +55,13 @@ global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
global RulesLogger logger global RulesLogger logger
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"
$fileAttribute: FileAttribute() $fileAttribute: FileAttribute()
end end
//------------------------------------ H rules ------------------------------------ //------------------------------------ Headlines rules ------------------------------------
// Rule unit: H.0 // Rule unit: H.0
rule "H.0.0: retract table of contents page" rule "H.0.0: retract table of contents page"
@ -129,6 +128,7 @@ rule "H.3.1: Study Type File Attribute in Headlines"
.ifPresent(fileAttribute -> insert(fileAttribute)); .ifPresent(fileAttribute -> insert(fileAttribute));
end end
//------------------------------------ General documine rules ------------------------------------ //------------------------------------ General documine rules ------------------------------------
// Rule unit: DOC.1 // Rule unit: DOC.1
@ -296,6 +296,7 @@ rule "DOC.1.4: Guideline in Headlines"
); );
end end
// Rule unit: DOC.2 // Rule unit: DOC.2
rule "DOC.2.0: Report number" rule "DOC.2.0: Report number"
when when
@ -1147,6 +1148,7 @@ rule "DOC.35.0: Doses (mg/kg bodyweight)"
.forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a")); .forEach(entity -> entity.apply("DOC.35.0", "Doses per bodyweight information found", "n-a"));
end end
//------------------------------------ Table extraction rules ------------------------------------ //------------------------------------ Table extraction rules ------------------------------------
// Rule unit: TAB.0 // Rule unit: TAB.0
@ -1296,7 +1298,8 @@ rule "TAB.7.0: Indicator (Species)"
.ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found")); .ifPresent(redactionEntity -> redactionEntity.apply("TAB.7.0", "Vertebrate study found"));
end end
//------------------------------------ Manual redaction rules ------------------------------------
//------------------------------------ Manual changes rules ------------------------------------
// Rule unit: MAN.0 // Rule unit: MAN.0
rule "MAN.0.0: Apply manual resize redaction" rule "MAN.0.0: Apply manual resize redaction"
@ -1424,6 +1427,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis()); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
@ -1485,7 +1489,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1534,8 +1537,6 @@ rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
retract($recommendation); retract($recommendation);
end end
// Rule unit: X.5
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION" rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
salience 256 salience 256
when when
@ -1580,6 +1581,7 @@ rule "X.8.1: Remove Entity when intersected by imported Entity"
retract($other); retract($other);
end end
// Rule unit: X.9 // Rule unit: X.9
rule "X.9.0: Merge mostly contained signatures" rule "X.9.0: Merge mostly contained signatures"
when when
@ -1590,6 +1592,7 @@ rule "X.9.0: Merge mostly contained signatures"
$signature.addEngine(LayoutEngine.AI); $signature.addEngine(LayoutEngine.AI);
end end
// Rule unit: X.10 // Rule unit: X.10
rule "X.10.0: remove false positives of ai" rule "X.10.0: remove false positives of ai"
when when

5
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_component_rules.drl
View File

@ -34,11 +34,6 @@ import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribu
global ComponentCreationService componentCreationService global ComponentCreationService componentCreationService
/**
The imports, globals, queries and rules from this file are required for any component rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"

8
redaction-service-v1/redaction-service-server-v1/src/main/resources/drools/base_rules.drl
View File

@ -61,12 +61,6 @@ global EntityCreationService entityCreationService
global ManualChangesApplicationService manualChangesApplicationService global ManualChangesApplicationService manualChangesApplicationService
global Dictionary dictionary global Dictionary dictionary
/**
The imports, globals, queries and rules from this file are required for any entity rule file.
Since customers may edit their rules we need to ensure they can't change the imports to prevent malicious code execution!
*/
//------------------------------------ queries ------------------------------------ //------------------------------------ queries ------------------------------------
query "getFileAttributes" query "getFileAttributes"
@ -75,7 +69,7 @@ query "getFileAttributes"
//------------------------------------ Local dictionary search rules ------------------------------------ //------------------------------------ Local dictionary search rules ------------------------------------
// Rule unit: LocalDictionarySearch.0 // Rule unit: LDS.0
rule "LDS.0.0: Run local dictionary search" rule "LDS.0.0: Run local dictionary search"
agenda-group "LOCAL_DICTIONARY_ADDS" agenda-group "LOCAL_DICTIONARY_ADDS"
salience -999 salience -999

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionAcceptanceTest.java
View File

@ -14,6 +14,7 @@ import java.util.UUID;
import java.util.stream.Stream; import java.util.stream.Stream;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -330,6 +331,7 @@ public class RedactionAcceptanceTest extends AbstractRedactionIntegrationTest {
@Test @Test
@SneakyThrows @SneakyThrows
@Disabled
void testNerEntitiesAfterReanalysis() { void testNerEntitiesAfterReanalysis() {
String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl"); String EFSA_SANITISATION_RULES = loadFromClassPath("drools/efsa_sanitisation.drl");

2
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/document/graph/DocumentPerformanceIntegrationTest.java
View File

@ -15,6 +15,7 @@ import java.util.List;
import org.apache.pdfbox.Loader; import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.kie.api.runtime.KieContainer; import org.kie.api.runtime.KieContainer;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -128,6 +129,7 @@ public class DocumentPerformanceIntegrationTest extends RulesIntegrationTest {
@Test @Test
@SneakyThrows @SneakyThrows
@Disabled
public void testBuildTextBlockPerformance() { public void testBuildTextBlockPerformance() {
int n = 10000; int n = 10000;

15
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/service/document/UnprocessedChangesServiceTest.java
View File

@ -1,5 +1,6 @@
package com.iqser.red.service.redaction.v1.server.service.document; package com.iqser.red.service.redaction.v1.server.service.document;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
@ -126,7 +127,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get(); UnprocessedManualEntity unprocessedManualEntity = optionalUnprocessedManualEntity.get();
assertEquals(unprocessedManualEntity.getTextBefore(), "was above the "); assertEquals(unprocessedManualEntity.getTextBefore(), "was above the ");
assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntity.getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntity.getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntity.getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
.get(0).x(), 355.53775f); .get(0).x(), 355.53775f);
assertEquals(unprocessedManualEntity.getPositions() assertEquals(unprocessedManualEntity.getPositions()
@ -173,7 +174,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " without PPE (34%");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to EFSA guidance ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -256,7 +257,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(resizedAoel.isPresent()); assertTrue(resizedAoel.isPresent());
assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of"); assertEquals(resizedAoel.get().getTextAfter(), " (max. 43% of");
assertEquals(resizedAoel.get().getTextBefore(), "is below the "); assertEquals(resizedAoel.get().getTextBefore(), "is below the ");
assertEquals(resizedAoel.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(resizedAoel.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(resizedAoel.get().getPositions() assertEquals(resizedAoel.get().getPositions()
@ -272,7 +273,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(cormsResized.isPresent()); assertTrue(cormsResized.isPresent());
assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of"); assertEquals(cormsResized.get().getTextAfter(), " a NOAEL of");
assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore "); assertEquals(cormsResized.get().getTextBefore(), "mg/kg bw/d. Furthermore ");
assertEquals(cormsResized.get().getSection(), "[0, 3]: Paragraph: The Co-RMS indicated the"); assertThat(cormsResized.get().getSection()).contains("Paragraph: The Co-RMS indicated the");
assertEquals(cormsResized.get().getPositions() assertEquals(cormsResized.get().getPositions()
.get(0).x(), positions2.get(0).getTopLeftX()); .get(0).x(), positions2.get(0).getTopLeftX());
assertEquals(cormsResized.get().getPositions() assertEquals(cormsResized.get().getPositions()
@ -288,7 +289,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertTrue(a9Resized.isPresent()); assertTrue(a9Resized.isPresent());
assertEquals(a9Resized.get().getTextAfter(), " were obtained from"); assertEquals(a9Resized.get().getTextAfter(), " were obtained from");
assertEquals(a9Resized.get().getTextBefore(), "data for S"); assertEquals(a9Resized.get().getTextBefore(), "data for S");
assertEquals(a9Resized.get().getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(a9Resized.get().getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
.get(0).x(), positions3.get(0).getTopLeftX()); .get(0).x(), positions3.get(0).getTopLeftX());
assertEquals(a9Resized.get().getPositions() assertEquals(a9Resized.get().getPositions()
@ -338,7 +339,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), " (max. 43% of");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "is below the ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[1, 1, 0]: Paragraph: A9396G containing 960 g/L"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: A9396G containing 960 g/L");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
@ -388,7 +389,7 @@ public class UnprocessedChangesServiceTest extends AbstractRedactionIntegrationT
assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId); assertEquals(unprocessedManualEntities.get(0).getAnnotationId(), aoelId);
assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same"); assertEquals(unprocessedManualEntities.get(0).getTextAfter(), ", the same");
assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an "); assertEquals(unprocessedManualEntities.get(0).getTextBefore(), "to set an ");
assertEquals(unprocessedManualEntities.get(0).getSection(), "[0, 4]: Paragraph: With respect to the"); assertThat(unprocessedManualEntities.get(0).getSection()).contains("Paragraph: With respect to the");
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()
.get(0).x(), positions.get(0).getTopLeftX()); .get(0).x(), positions.get(0).getTopLeftX());
assertEquals(unprocessedManualEntities.get(0).getPositions() assertEquals(unprocessedManualEntities.get(0).getPositions()

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -874,7 +875,7 @@ rule "PII.10.0: Redact study director abbreviation (non vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -1443,7 +1444,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/efsa_sanitisation.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -611,7 +612,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));

3
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -844,7 +845,7 @@ rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));

21
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -388,6 +388,27 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY), !hasManualChanges())
then
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64
when
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.HINT), !hasManualChanges())
then
$entity.getIntersectingNodes().forEach(node -> update(node));
$entity.remove("X.2.1", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
retract($entity)
end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when

1108
redaction-service-v1/redaction-service-server-v1/src/test/resources/files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 3.NER_ENTITIES.json
View File

File diff suppressed because one or more lines are too long

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/logs/rules_logging.drl
View File

@ -4,7 +4,13 @@ import com.iqser.red.service.redaction.v1.server.logger.RulesLogger;
global RulesLogger logger global RulesLogger logger
//------------------------------------ queries ------------------------------------
//------------------------------------ LOG rules ------------------------------------
// Rule unit: LOG.0
rule "LOG.0.0: Test log info" rule "LOG.0.0: Test log info"
salience 1 salience 1
when when

6
redaction-service-v1/redaction-service-server-v1/src/test/resources/performance/dictionaries/EFSA_sanitisation_GFL_v1/rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -420,7 +421,6 @@ rule "PII.3.2: Redact telephone numbers by RegEx (vertebrate study)"
.forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002")); .forEach(entity -> entity.redact("PII.3.2", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end end
rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)" rule "PII.3.4: Redact telephone numbers by RegEx (Non vertebrate study)"
when when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y") not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
@ -593,7 +593,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
@ -901,7 +901,6 @@ rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
retract($entity) retract($entity)
end end
rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE" rule "X.2.1: Remove Entity of type HINT when contained by FALSE_POSITIVE"
salience 64 salience 64
when when
@ -1040,7 +1039,6 @@ rule "X.11.1: Remove non manual entity which intersects with a manual entity"
retract($nonManualEntity); retract($nonManualEntity);
end end
rule "X.11.2: Remove non manual entity which are equal to manual entity" rule "X.11.2: Remove non manual entity which are equal to manual entity"
salience 70 salience 70
when when

3
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -22,6 +22,7 @@ import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntit
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*; import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SuperSection;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table; import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell; import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode; import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
@ -1458,7 +1459,7 @@ rule "PII.10.1: Redact study director abbreviation (vertebrate study)"
// Rule unit: PII.11 // Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:" rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title")) $section: SuperSection(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section) entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002")); .forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));