RED-8305: Apply recategorizations by default, increase memory for unittests #256

Merged
dominique.eiflaender1 merged 1 commits from RED-8305 into master 2024-01-22 14:22:45 +01:00
13 changed files with 434 additions and 50 deletions
Showing only changes of commit c71241288c - Show all commits

View File

@ -77,9 +77,13 @@ tasks.test {
configure<JacocoTaskExtension> { configure<JacocoTaskExtension> {
excludes = listOf("org/drools/**/*") excludes = listOf("org/drools/**/*")
} }
minHeapSize = "512m"
maxHeapSize = "1024m"
} }
tasks.named<BootBuildImage>("bootBuildImage") { tasks.named<BootBuildImage>("bootBuildImage") {
environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ") environment.put("BPE_DELIM_JAVA_TOOL_OPTIONS", " ")

View File

@ -1223,6 +1223,56 @@ public class RedactionIntegrationTest extends AbstractRedactionIntegrationTest {
} }
@Test
public void entityIsAppliedAfterRecateorize() throws IOException {
AnalyzeRequest request = uploadFileToStorage("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).pdf");
ClassPathResource imageServiceResponseFileResource = new ClassPathResource("files/new/SYNGENTA_EFSA_sanitisation_GFL_v1 (1).IMAGE_INFO.json");
storageService.storeObject(TenantContext.getTenantId(),
RedactionStorageService.StorageIdUtils.getStorageId(TEST_DOSSIER_ID, TEST_FILE_ID, FileType.IMAGE_INFO),
imageServiceResponseFileResource.getInputStream());
System.out.println("Start Full integration test");
analyzeDocumentStructure(LayoutParsingType.REDACT_MANAGER, request);
System.out.println("Finished structure analysis");
analyzeService.analyze(request);
System.out.println("Finished analysis");
request.setManualRedactions(ManualRedactions.builder()
.legalBasisChanges(Set.of(ManualLegalBasisChange.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.section("[19, 2]: Paragraph: Contact point: LexCo Contact:")
.value("0049 331 441 551 14")
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.legalBasis("Article 39(e)(2) of Regulation (EC) No 178/2002")
.build()))
.recategorizations(Set.of(ManualRecategorization.builder()
.annotationId("3029651d0842a625f2d23f8375c23600")
.type("CBI_author")
.requestDate(OffsetDateTime.now())
.status(AnnotationStatus.APPROVED)
.fileId(TEST_FILE_ID)
.build()))
.build());
analyzeService.reanalyze(request);
System.out.println("Finished reanalysis");
var entityLog = redactionStorageService.getEntityLog(TEST_DOSSIER_ID, TEST_FILE_ID);
var changedAnnotation = entityLog.getEntityLogEntry()
.stream()
.filter(entityLogEntry -> entityLogEntry.getId().equals("3029651d0842a625f2d23f8375c23600"))
.findFirst()
.get();
assertThat(changedAnnotation.getState() == EntryState.APPLIED).isTrue();
}
@Test @Test
@SneakyThrows @SneakyThrows
public void testRemovingAndAddingRedactionOnTheSameValue() { public void testRemovingAndAddingRedactionOnTheSameValue() {

View File

@ -374,6 +374,214 @@ rule "PII.2.1: Redact Phone and Fax by RegEx (vertebrate study)"
end end
// Rule unit: PII.3
rule "PII.3.0: Redact telephone numbers by RegEx (Non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.0", "Telephone number found by regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.3.1: Redact telephone numbers by RegEx (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(matchesRegex("[+]\\d{1,}"))
then
entityCreationService.byRegex("((([+]\\d{1,3} (\\d{7,12})\\b)|([+]\\d{1,3}(\\d{3,12})\\b|[+]\\d{1,3}([ -]\\(?\\d{1,6}\\)?){2,4})|[+]\\d{1,3} ?((\\d{2,6}\\)?)([ -]\\d{2,6}){1,4}))(-\\d{1,3})?\\b)", "PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.3.1", "Telephone number found by regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.4
rule "PII.4.0: Redact line after contact information keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.4.1: Redact line after contact information keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:",
"No:",
"Contact:",
"Tel.:",
"Tel:",
"Telephone number:",
"Telephone No:",
"Telephone:",
"Phone No.",
"Phone:",
"Fax number:",
"Fax:",
"E-mail:",
"Email:",
"e-mail:",
"E-mail address:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
// Rule unit: PII.5
rule "PII.5.0: Redact line after contact information keywords reduced (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.5.1: Redact line after contact information keywords reduced (Vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$contactKeyword: String() from List.of("Contact point:",
"Contact:",
"Alternative contact:",
"European contact:")
$section: Section(containsString($contactKeyword))
then
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
.forEach(contactEntity -> contactEntity.redact("PII.5.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.6
rule "PII.6.0: Redact line between contact keywords (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.6.1: Redact line between contact keywords (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
then
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
)
.forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.7
rule "PII.7.0: Redact contact information if applicant is found (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(getHeadline().containsString("applicant") ||
getHeadline().containsString("Primary contact") ||
getHeadline().containsString("Alternative contact") ||
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.7.1: Redact contact information if applicant is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(getHeadline().containsString("applicant") ||
getHeadline().containsString("Primary contact") ||
getHeadline().containsString("Alternative contact") ||
containsString("Applicant") ||
containsString("Telephone number:"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.8
rule "PII.8.0: Redact contact information if producer is found (non vertebrate study)"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
containsStringIgnoreCase("producer of the active substance") ||
containsStringIgnoreCase("manufacturer of the active substance") ||
containsStringIgnoreCase("manufacturer:") ||
containsStringIgnoreCase("Producer or producers of the active substance"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
end
rule "PII.8.1: Redact contact information if producer is found (vertebrate study)"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
containsStringIgnoreCase("producer of the active substance") ||
containsStringIgnoreCase("manufacturer of the active substance") ||
containsStringIgnoreCase("manufacturer:") ||
containsStringIgnoreCase("Producer or producers of the active substance"))
then
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
Stream.concat(
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
))
.forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.9 // Rule unit: PII.9
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)" rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
when when
@ -394,6 +602,46 @@ rule "PII.9.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (ver
end end
// Rule unit: PII.10
rule "PII.10.0: Redact study director abbreviation"
when
$section: Section(containsString("KATH") || containsString("BECH") || containsString("KML"))
then
entityCreationService.byRegexIgnoreCase("((KATH)|(BECH)|(KML)) ?(\\d{4})","PII", EntityType.ENTITY, 1, $section)
.forEach(entity -> entity.redact("PII.10.0", "Personal information found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.11
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
when
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
then
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
// Rule unit: PII.12
rule "PII.12.0: Expand PII entities with salutation prefix"
when
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
end
rule "PII.12.1: Expand PII entities with salutation prefix"
when
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
$entityToExpand: TextEntity(type == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
then
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
end
//------------------------------------ Other rules ------------------------------------ //------------------------------------ Other rules ------------------------------------
// Rule unit: ETC.0 // Rule unit: ETC.0
@ -597,15 +845,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -615,6 +873,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -713,8 +973,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -1415,15 +1415,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -1433,6 +1443,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -1531,8 +1543,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -1278,7 +1278,7 @@ rule "MAN.3.3: Apply recategorization entities by default"
when when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default."); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
@ -1286,10 +1286,12 @@ rule "MAN.3.3: Apply recategorization entities by default"
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -1299,6 +1301,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end

View File

@ -187,15 +187,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -205,6 +215,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end

View File

@ -1005,22 +1005,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.4: Apply recategorization with legalbasis change by default" rule "MAN.3.3: Apply recategorization entities by default"
salience 128 salience 128
when when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent()) $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then then
$entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default."); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -1030,6 +1033,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -1128,8 +1133,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -211,15 +211,25 @@ rule "MAN.3.1: Apply entity recategorization of same type"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -229,6 +239,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -327,8 +339,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -337,15 +337,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -355,6 +365,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -437,8 +449,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -237,15 +237,25 @@ rule "MAN.3.2: Apply image recategorization"
retract($recategorization); retract($recategorization);
end end
rule "MAN.3.3: Apply recategorization entities by default"
salience 128
when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -255,6 +265,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -353,8 +365,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -1437,22 +1437,24 @@ rule "MAN.3.2: Apply image recategorization"
end end
rule "MAN.3.4: Apply recategorization with legalbasis change by default" rule "MAN.3.3: Apply recategorization entities by default"
salience 128 salience 128
when when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false), getManualOverwrite().getLegalBasis().isPresent()) $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then then
$entity.apply("MAN.3.4", "Recategorized entities with legalbasis are applied by default."); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -1462,6 +1464,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end
@ -1561,8 +1565,8 @@ rule "X.6.0: Remove Entity of lower rank, when contained by by entity of type EN
rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity" rule "X.6.1: remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"
salience 32 salience 32
when when
$higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active()) $higherRank: TextEntity($type: type, $value: value, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active(), !hasManualChanges())
$lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges(), active(), $lowerRank.getValue().length() > $value.length()) $lowerRank: TextEntity(intersects($higherRank), type != $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), active(), $lowerRank.getValue().length() > $value.length())
then then
$higherRank.getIntersectingNodes().forEach(node -> update(node)); $higherRank.getIntersectingNodes().forEach(node -> update(node));
$higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity"); $higherRank.remove("X.6.1", "remove Entity of higher rank, when intersected by entity of type ENTITY and length of lower rank Entity is bigger than the higher rank Entity");

View File

@ -1425,17 +1425,19 @@ rule "MAN.3.3: Apply recategorization entities by default"
when when
$entity: IEntity(getManualOverwrite().getRecategorized().orElse(false)) $entity: IEntity(getManualOverwrite().getRecategorized().orElse(false))
then then
$entity.apply("MAN.3.3", "Recategorized entities are applied by default."); $entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
end end
// Rule unit: MAN.4 // Rule unit: MAN.4
rule "MAN.4.0: Apply legal basis change" rule "MAN.4.0: Apply legal basis change"
salience 128 salience 128
when when
$legalbasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED) $legalBasisChange: ManualLegalBasisChange($id: annotationId, status == AnnotationStatus.APPROVED)
$imageToBeRecategorized: Image($id == id) $imageToBeRecategorized: Image($id == id)
then then
$imageToBeRecategorized.getManualOverwrite().addChange($legalbasisChange); $imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
update($imageToBeRecategorized)
retract($legalBasisChange)
end end
rule "MAN.4.1: Apply legal basis change" rule "MAN.4.1: Apply legal basis change"
@ -1445,6 +1447,8 @@ rule "MAN.4.1: Apply legal basis change"
$entityToBeChanged: TextEntity(matchesAnnotationId($id)) $entityToBeChanged: TextEntity(matchesAnnotationId($id))
then then
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange); $entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
update($entityToBeChanged)
retract($legalBasisChange)
end end

View File

@ -1,11 +1,4 @@
MAN.0.* MAN.*.*
MAN.1.*
MAN.2.*
MAN.3.0
MAN.3.1
MAN.3.2
MAN.3.4
MAN.4.*
X.0.0 X.0.0
X.1.0 X.1.0
X.2.0 X.2.0