Marmelator/redaction-service
1
0
Fork 0
Code Issues Pull Requests 14 Actions Packages Projects Releases 622 Wiki Activity

RED-8305: Apply recategorizations by default, increase memory for unittests #256

Merged
dominique.eiflaender1 merged 1 commits from RED-8305 into master 2024-01-22 14:22:45 +01:00
Conversation 4 Commits 1 Files Changed 13 +434 -50
13 changed files with 434 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
redaction-service-v1/redaction-service-server-v1/build.gradle.kts
View File

@ -77,9 +77,13 @@ tasks.test {
configure<JacocoTaskExtension> {
excludes = listOf("org/drools/**/*")
}
minHeapSize = "512m"
maxHeapSize = "1024m"
}
tasks.named<BootBuildImage>("bootBuildImage") {
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")

50
redaction-service-v1/redaction-service-server-v1/src/test/java/com/iqser/red/service/redaction/v1/server/RedactionIntegrationTest.java
View File

@ -1223,6 +1223,56 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
}
@Test
public void entityIsAppliedAfterRecateorize() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
analyzeService.analyze(request);
System.out.println("Finished analysis");
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.type("CBI_author")
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.build()))
.build());
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changedAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
.findFirst()
.get();
assertThat(changedAnnotation.getState() == EntryState.APPLIED).isTrue();
}
@Test
@SneakyThrows
public void testRemovingAndAddingRedactionOnTheSameValue() {

268
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/acceptance_rules.drl
View File

@ -374,6 +374,214 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
end
// Rule unit: PII.3
rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.4
rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
// Rule unit: PII.5
rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.6
rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.7
rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(getHeadline().containsString("applicant") ||
getHeadline().containsString("Primary contact") ||
getHeadline().containsString("Alternative contact") ||
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(getHeadline().containsString("applicant") ||
getHeadline().containsString("Primary contact") ||
getHeadline().containsString("Alternative contact") ||
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.8
rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
containsStringIgnoreCase("producer of the active substance") ||
containsStringIgnoreCase("manufacturer of the active substance") ||
containsStringIgnoreCase("manufacturer:") ||
containsStringIgnoreCase("Producer or producers of the active substance"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
containsStringIgnoreCase("producer of the active substance") ||
containsStringIgnoreCase("manufacturer of the active substance") ||
containsStringIgnoreCase("manufacturer:") ||
containsStringIgnoreCase("Producer or producers of the active substance"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when
@ -394,6 +602,46 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver
end
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
when
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.12
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
//------------------------------------ Other rules ------------------------------------
// Rule unit: ETC.0
@ -597,15 +845,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -615,6 +873,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -713,8 +973,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

20
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/all_redact_manager_rules.drl
View File

@ -1415,15 +1415,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -1433,6 +1443,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -1531,8 +1543,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

10
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/documine_flora.drl
View File

@ -1278,7 +1278,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.");
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
@ -1286,10 +1286,12 @@ rule "MAN.3.3: Apply recategorization entities by default"
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -1299,6 +1301,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end

16
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/manual_redaction_rules.drl
View File

@ -187,15 +187,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -205,6 +215,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end

19
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules.drl
View File

@ -1005,22 +1005,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.4: Apply recategorization with legalbasis change by default"
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent())
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default.");
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -1030,6 +1033,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -1128,8 +1133,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

20
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/rules_v2.drl
View File

@ -211,15 +211,25 @@ rule "MAN.3.1: Apply entity recategorization of same type"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -229,6 +239,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -327,8 +339,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

20
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/table_demo.drl
View File

@ -337,15 +337,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -355,6 +365,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -437,8 +449,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

20
redaction-service-v1/redaction-service-server-v1/src/test/resources/drools/test_rules.drl
View File

@ -237,15 +237,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization);
end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -255,6 +265,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -353,8 +365,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

18
redaction-service-v1/rules-management/src/main/resources/all_redact_manager_rules.drl
View File

@ -1437,22 +1437,24 @@ rule "MAN.3.2: Apply image recategorization"
end
rule "MAN.3.4: Apply recategorization with legalbasis change by default"
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent())
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default.");
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -1462,6 +1464,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end
@ -1561,8 +1565,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32
when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length())
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then
$higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

10
redaction-service-v1/rules-management/src/main/resources/all_rules_documine.drl
View File

@ -1425,17 +1425,19 @@ rule "MAN.3.3: Apply recategorization entities by default"
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.");
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change"
salience 128
when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id)
then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange);
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end
rule "MAN.4.1: Apply legal basis change"
@ -1445,6 +1447,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end

9
redaction-service-v1/rules-management/src/main/resources/default_rule_identifiers.txt
View File

@ -1,11 +1,4 @@
MAN.0.*
MAN.1.*
MAN.2.*
MAN.3.0
MAN.3.1
MAN.3.2
MAN.3.4
MAN.4.*
MAN.*.*
X.0.0
X.1.0
X.2.0