1536 lines
77 KiB
Plaintext
1536 lines
77 KiB
Plaintext
package drools
|
|
|
|
import static java.lang.String.format;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.anyMatch;
|
|
import static com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility.exactMatch;
|
|
|
|
import java.util.List;
|
|
import java.util.LinkedList;
|
|
import java.util.Set;
|
|
import java.util.stream.Collectors;
|
|
import java.util.Collection;
|
|
import java.util.stream.Stream;
|
|
import java.util.Optional;
|
|
|
|
import com.iqser.red.service.redaction.v1.server.model.document.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.TextRange;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.EntityType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.TextEntity
|
|
import com.iqser.red.service.redaction.v1.server.model.document.entity.MatchedRule
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Section;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Table;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.TableCell;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SemanticNode;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Document;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Paragraph;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Image;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.ImageType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Page;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Headline;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.SectionIdentifier;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Footer;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.Header;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.nodes.NodeType;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.*;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.TextBlockCollector;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.AtomicTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.document.textblock.ConcatenatedTextBlock;
|
|
import com.iqser.red.service.redaction.v1.server.model.NerEntities;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.Dictionary;
|
|
import com.iqser.red.service.redaction.v1.server.model.dictionary.DictionaryModel;
|
|
import com.iqser.red.service.redaction.v1.server.service.document.EntityCreationService;
|
|
import com.iqser.red.service.redaction.v1.server.service.ManualChangesApplicationService;
|
|
import com.iqser.red.service.redaction.v1.server.utils.RedactionSearchUtility;
|
|
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.FileAttribute;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.analysislog.entitylog.Engine;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualResizeRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.IdRemoval;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualForceRedaction;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualRecategorization;
|
|
import com.iqser.red.service.persistence.service.v1.api.shared.model.annotations.entitymapped.ManualLegalBasisChange;
|
|
|
|
import com.knecon.fforesight.service.layoutparser.internal.api.data.redaction.LayoutEngine;
|
|
|
|
global Document document
|
|
global EntityCreationService entityCreationService
|
|
global ManualChangesApplicationService manualChangesApplicationService
|
|
global Dictionary dictionary
|
|
|
|
//------------------------------------ queries ------------------------------------
|
|
|
|
query "getFileAttributes"
|
|
$fileAttribute: FileAttribute()
|
|
end
|
|
|
|
//------------------------------------ Syngenta specific rules ------------------------------------
|
|
|
|
// Rule unit: SYN.0
|
|
rule "SYN.0.0: Redact if CTL/* or BL/* was found (Non Vertebrate Study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("CTL/") || containsString("BL/"))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.byString("CTL", "must_redact", EntityType.HINT, $section),
|
|
entityCreationService.byString("BL", "must_redact", EntityType.HINT, $section)
|
|
).forEach(entity -> entity.skip("SYN.0.0", "hint_only"));
|
|
end
|
|
|
|
|
|
//------------------------------------ CBI rules ------------------------------------
|
|
|
|
// Rule unit: CBI.3
|
|
rule "CBI.3.0: Redacted because Section contains a vertebrate"
|
|
when
|
|
$section: Section(!hasTables(), hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.3.0",
|
|
"Vertebrate found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
$section.getEntitiesOfType("vertebrate")
|
|
);
|
|
});
|
|
end
|
|
|
|
rule "CBI.3.1: Redacted because table row contains a vertebrate"
|
|
when
|
|
$table: Table(hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate"))
|
|
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.3.1",
|
|
"Vertebrate found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
$table.getEntitiesOfTypeInSameRow("vertebrate", entity)
|
|
);
|
|
});
|
|
end
|
|
|
|
rule "CBI.3.2: Do not redact because Section does not contain a vertebrate"
|
|
when
|
|
$section: Section(!hasTables(), !hasEntitiesOfType("vertebrate"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(entity -> entity.skip("CBI.3.2", "No vertebrate found"));
|
|
end
|
|
|
|
rule "CBI.3.3: Do not redact because table row does not contain a vertebrate"
|
|
when
|
|
$table: Table(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address"))
|
|
then
|
|
$table.streamEntitiesWhereRowContainsNoEntitiesOfType(List.of("vertebrate"))
|
|
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
|
|
.forEach(entity -> entity.skip("CBI.3.3", "No vertebrate found"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.4
|
|
rule "CBI.4.0: Do not redact Names and Addresses if no_redaction_indicator is found in Section"
|
|
when
|
|
$section: Section(!hasTables(),
|
|
hasEntitiesOfType("vertebrate"),
|
|
hasEntitiesOfType("no_redaction_indicator"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.skipWithReferences(
|
|
"CBI.4.0",
|
|
"Vertebrate but a no redaction indicator found",
|
|
$section.getEntitiesOfType("no_redaction_indicator")
|
|
);
|
|
});
|
|
end
|
|
|
|
rule "CBI.4.1: Don't redact authors or addresses which appear in the same row as a vertebrate and a no_redaction_indicator"
|
|
when
|
|
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
|
|
hasEntitiesOfType("vertebrate"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("vertebrate", "no-redaction_indicator"))
|
|
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.skipWithReferences(
|
|
"CBI.4.1",
|
|
"Vertebrate but a no redaction indicator found",
|
|
Stream.concat(
|
|
$table.getEntitiesOfTypeInSameRow("vertebrate", entity).stream(),
|
|
$table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
|
|
);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.5
|
|
rule "CBI.5.0: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in Section"
|
|
when
|
|
$section: Section(!hasTables(),
|
|
hasEntitiesOfType("redaction_indicator"),
|
|
hasEntitiesOfType("no_redaction_indicator"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.5.0",
|
|
"no_redaction_indicator but also redaction_indicator found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
Stream.concat(
|
|
$section.getEntitiesOfType("vertebrate").stream(),
|
|
$section.getEntitiesOfType("no_redaction_indicator").stream()).toList()
|
|
);
|
|
});
|
|
end
|
|
|
|
rule "CBI.5.1: Redact Names and Addresses if no_redaction_indicator but also redaction_indicator is found in table row"
|
|
when
|
|
$table: Table(hasEntitiesOfType("no_redaction_indicator"),
|
|
hasEntitiesOfType("redaction_indicator"),
|
|
(hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("redaction_indicator", "no_redaction_indicator"))
|
|
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.5.1",
|
|
"no_redaction_indicator but also redaction_indicator found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
Stream.concat(
|
|
$table.getEntitiesOfTypeInSameRow("redaction_indicator", entity).stream(),
|
|
$table.getEntitiesOfTypeInSameRow("no_redaction_indicator", entity).stream()).toList()
|
|
);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.8
|
|
rule "CBI.8.0: Redacted because Section contains must_redact entity"
|
|
when
|
|
$section: Section(!hasTables(), hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.8.0",
|
|
"must_redact entity found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
$section.getEntitiesOfType("must_redact")
|
|
);
|
|
});
|
|
end
|
|
|
|
rule "CBI.8.1: Redacted because table row contains must_redact entity"
|
|
when
|
|
$table: Table(hasEntitiesOfType("must_redact"), (hasEntitiesOfType("CBI_author") || hasEntitiesOfType("CBI_address")))
|
|
then
|
|
$table.streamEntitiesWhereRowContainsEntitiesOfType(List.of("must_redact"))
|
|
.filter(entity -> entity.getType().equals("CBI_author") || entity.getType().equals("CBI_address"))
|
|
.forEach(entity -> {
|
|
entity.applyWithReferences(
|
|
"CBI.8.1",
|
|
"Must_redact found",
|
|
"Reg (EC) No 1107/2009 Art. 63 (2g)",
|
|
$table.getEntitiesOfTypeInSameRow("must_redact", entity)
|
|
);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.9
|
|
rule "CBI.9.0: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$table: Table(hasHeader("Author(s)"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Author(s)")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
|
.filter(Optional::isPresent)
|
|
.map(Optional::get)
|
|
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.0", "Author(s) found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "CBI.9.1: Redact all cells with Header Author as CBI_author (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$table: Table(hasHeader("Author"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Author")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
|
.filter(Optional::isPresent)
|
|
.map(Optional::get)
|
|
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.1", "Author found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "CBI.9.2: Redact all cells with Header Author(s) as CBI_author (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$table: Table(hasHeader("Author(s)"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Author(s)")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "CBI_author", EntityType.ENTITY))
|
|
.filter(Optional::isPresent)
|
|
.map(Optional::get)
|
|
.forEach(redactionEntity -> redactionEntity.redact("CBI.9.2", "Author(s) found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.11
|
|
rule "CBI.11.0: Recommend all CBI_author entities in Table with Vertebrate Study Y/N Header"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -1
|
|
when
|
|
$table: Table(hasHeader("Author(s)") && hasHeader("Vertebrate Study Y/N"))
|
|
then
|
|
$table.getEntitiesOfType("CBI_author").stream().filter(IEntity::applied).forEach(entity -> dictionary.addMultipleAuthorsAsRecommendation(entity));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.12
|
|
rule "CBI.12.0: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes'"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
|
|
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
|
|
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
|
|
$rowCell: TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
|
|
TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
|
|
then
|
|
entityCreationService.bySemanticNode($rowCell, "must_redact", EntityType.HINT)
|
|
.ifPresent(yesEntity -> {
|
|
yesEntity.skip("CBI.12.0", "must_redact");
|
|
});
|
|
end
|
|
|
|
rule "CBI.12.1: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
|
|
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
|
|
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
|
|
TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
|
|
$authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
|
|
then
|
|
entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
|
|
.ifPresent(authorEntity -> {
|
|
authorEntity.redact("CBI.12.1", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
|
|
});
|
|
|
|
end
|
|
|
|
rule "CBI.12.2: Redact and recommend TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'Yes' (vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
|
|
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
|
|
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
|
|
TableCell(!header, containsAnyString("Yes", "Y"), $rowWithYes: row) from $table.streamCol($vertebrateCol).toList()
|
|
$authorCell: TableCell(row == $rowWithYes) from $table.streamCol($authorCol).toList()
|
|
then
|
|
|
|
entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
|
|
.ifPresent(authorEntity -> {
|
|
authorEntity.redact("CBI.12.2", "Redacted because it's row belongs to a vertebrate study", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
dictionary.addMultipleAuthorsAsRecommendation(authorEntity);
|
|
});
|
|
|
|
end
|
|
|
|
rule "CBI.12.3: Skip TableCell with header 'Author' or 'Author(s)' and header 'Vertebrate study Y/N' with value 'No'"
|
|
when
|
|
$table: Table(hasHeader("Author(s)") || hasHeader("Author"), hasHeaderIgnoreCase("Vertebrate Study Y/N"))
|
|
TableCell(header, containsAnyStringIgnoreCase("Author", "Author(s)"), $authorCol: col) from $table.streamHeaders().toList()
|
|
TableCell(header, containsStringIgnoreCase("Vertebrate study Y/N"), $vertebrateCol: col) from $table.streamHeaders().toList()
|
|
TableCell(!header, containsAnyString("No", "N"), $rowWithNo: row) from $table.streamCol($vertebrateCol).toList()
|
|
$authorCell: TableCell(row == $rowWithNo) from $table.streamCol($authorCol).toList()
|
|
then
|
|
entityCreationService.bySemanticNode($authorCell, "CBI_author", EntityType.ENTITY)
|
|
.ifPresent(authorEntity -> authorEntity.skip("CBI.12.3", "Not redacted because it's row does not belong to a vertebrate study"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.14
|
|
rule "CBI.14.0: Redact CBI_sponsor entities if preceded by \"batches produced at\""
|
|
when
|
|
$section: Section(containsStringIgnoreCase("batches produced at"))
|
|
$sponsorEntity: TextEntity(type() == "CBI_sponsor", textBefore.contains("batches produced at"))
|
|
then
|
|
$sponsorEntity.redact("CBI.14.0", "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
entityCreationService.byString("batches produced at", "must_redact", EntityType.HINT, $section)
|
|
.forEach(entity -> entity.skip("CBI.14.0", "must_redact"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.15
|
|
rule "CBI.15.0: Redact row if row contains \"determination of residues\" and livestock keyword"
|
|
when
|
|
$keyword: String() from List.of("livestock",
|
|
"live stock",
|
|
"tissue",
|
|
"tissues",
|
|
"liver",
|
|
"muscle",
|
|
"bovine",
|
|
"ruminant",
|
|
"ruminants")
|
|
$residueKeyword: String() from List.of("determination of residues", "determination of total residues")
|
|
$section: Section(!hasTables(),
|
|
containsStringIgnoreCase($residueKeyword),
|
|
containsStringIgnoreCase($keyword))
|
|
then
|
|
entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $section)
|
|
.forEach(entity -> entity.skip("CBI.15.0", "must_redact"));
|
|
|
|
$section.getEntitiesOfType(List.of("CBI_author", "CBI_address"))
|
|
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.0", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
|
|
end
|
|
|
|
rule "CBI.15.1: Redact CBI_author and CBI_address if row contains \"determination of residues\" and livestock keyword"
|
|
when
|
|
$keyword: String() from List.of("livestock",
|
|
"live stock",
|
|
"tissue",
|
|
"tissues",
|
|
"liver",
|
|
"muscle",
|
|
"bovine",
|
|
"ruminant",
|
|
"ruminants")
|
|
$residueKeyword: String() from List.of("determination of residues", "determination of total residues")
|
|
$table: Table(containsStringIgnoreCase($residueKeyword), containsStringIgnoreCase($keyword))
|
|
then
|
|
entityCreationService.byString($keyword, "must_redact", EntityType.HINT, $table)
|
|
.forEach(entity -> entity.skip("CBI.15.1", "must_redact"));
|
|
|
|
$table.streamEntitiesWhereRowContainsStringsIgnoreCase(List.of($keyword, $residueKeyword))
|
|
.filter(redactionEntity -> redactionEntity.isAnyType(List.of("CBI_author", "CBI_address")))
|
|
.forEach(redactionEntity -> redactionEntity.redact("CBI.15.1", "Determination of residues and keyword \"" + $keyword + "\" was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.16
|
|
rule "CBI.16.0: Add CBI_author with \"et al.\" RegEx"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.16.0", "Author found by \"et al\" regex", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.16.1: Add CBI_author with \"et al.\" RegEx (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.16.1", "Author found by \"et al\" regex", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.16.2: Add CBI_author with \"et al.\" RegEx (vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.16.2", "Author found by \"et al\" regex", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.16.3: Add CBI_author with \"et al.\" RegEx"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: Section(containsString("et al."))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", "CBI_author", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> {
|
|
entity.redact("CBI.16.3", "Author found by \"et al\" regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
|
|
dictionary.recommendEverywhere(entity);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.17
|
|
rule "CBI.17.0: Add recommendation for Addresses in Test Organism sections, without colon"
|
|
when
|
|
$section: Section(!hasTables(), containsString("Species") && containsString("Source") && !containsString("Species:") && !containsString("Source:"))
|
|
then
|
|
entityCreationService.lineAfterString("Source", "CBI_address", EntityType.RECOMMENDATION, $section)
|
|
.forEach(entity -> entity.skip("CBI.17.0", "Line after \"Source\" in Test Organism Section"));
|
|
end
|
|
|
|
rule "CBI.17.1: Add recommendation for Addresses in Test Organism sections, with colon"
|
|
when
|
|
$section: Section(!hasTables(), containsString("Species:"), containsString("Source:"))
|
|
then
|
|
entityCreationService.lineAfterString("Source:", "CBI_address", EntityType.RECOMMENDATION, $section)
|
|
.forEach(entity -> entity.skip("CBI.17.1", "Line after \"Source:\" in Test Animals Section"));
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.18
|
|
rule "CBI.18.0: Expand CBI_author entities with firstname initials"
|
|
no-loop true
|
|
when
|
|
$entityToExpand: TextEntity(type() == "CBI_author",
|
|
value.matches("[^\\s]+"),
|
|
textAfter.startsWith(" "),
|
|
anyMatch(textAfter, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
|
|
)
|
|
then
|
|
entityCreationService.bySuffixExpansionRegex($entityToExpand, "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)")
|
|
.ifPresent(expandedEntity -> {
|
|
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
|
|
$entityToExpand.remove("CBI.18.0", "Expand CBI_author entities with firstname initials");
|
|
retract($entityToExpand);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.19
|
|
rule "CBI.19.0: Expand CBI_author entities with salutation prefix"
|
|
when
|
|
$entityToExpand: TextEntity(type() == "CBI_author", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
|
|
then
|
|
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
|
|
.ifPresent(expandedEntity -> {
|
|
expandedEntity.addMatchedRules($entityToExpand.getMatchedRuleList());
|
|
$entityToExpand.remove("CBI.19.0", "Expand CBI_author entities with salutation prefix");
|
|
retract($entityToExpand);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.20
|
|
rule "CBI.20.0: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\")"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
|
then
|
|
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
|
.forEach(laboratoryEntity -> {
|
|
laboratoryEntity.redact("CBI.20.0", "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
dictionary.recommendEverywhere(laboratoryEntity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.20.1: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (non vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
|
then
|
|
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
|
.forEach(laboratoryEntity -> {
|
|
laboratoryEntity.skip("CBI.20.1", "PERFORMING LABORATORY was found for non vertebrate study");
|
|
dictionary.recommendEverywhere(laboratoryEntity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.20.2: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\" (vertebrate study)"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
|
then
|
|
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
|
.forEach(laboratoryEntity -> {
|
|
laboratoryEntity.redact("CBI.20.2", "PERFORMING LABORATORY was found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
dictionary.recommendEverywhere(laboratoryEntity);
|
|
});
|
|
end
|
|
|
|
rule "CBI.20.3: Redact between \"PERFORMING LABORATORY\" and \"LABORATORY PROJECT ID:\""
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
when
|
|
$section: Section(!hasTables(), containsString("PERFORMING LABORATORY:"), containsString("LABORATORY PROJECT ID:"))
|
|
then
|
|
entityCreationService.betweenStrings("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "CBI_address", EntityType.ENTITY, $section)
|
|
.forEach(laboratoryEntity -> {
|
|
laboratoryEntity.redact("CBI.20.3", "PERFORMING LABORATORY was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
|
|
dictionary.recommendEverywhere(laboratoryEntity);
|
|
});
|
|
end
|
|
|
|
|
|
// Rule unit: CBI.23
|
|
rule "CBI.23.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
|
|
then
|
|
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
|
|
.forEach(authorEntity -> authorEntity.redact("CBI.23.0", "AUTHOR(S) was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "CBI.23.1: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\" (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
|
|
then
|
|
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "CBI_author", EntityType.ENTITY, $document)
|
|
.forEach(authorEntity -> authorEntity.redact("CBI.23.1", "AUTHOR(S) was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
//------------------------------------ PII rules ------------------------------------
|
|
|
|
// Rule unit: PII.0
|
|
rule "PII.0.0: Redact all PII"
|
|
when
|
|
$pii: TextEntity(type() == "PII", dictionaryEntry)
|
|
then
|
|
$pii.redact("PII.0.0", "Personal Information found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "PII.0.1: Redact all PII (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$pii: TextEntity(type() == "PII", dictionaryEntry)
|
|
then
|
|
$pii.redact("PII.0.1", "Personal Information found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "PII.0.2: Redact all PII (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$pii: TextEntity(type() == "PII", dictionaryEntry)
|
|
then
|
|
$pii.redact("PII.0.2", "Personal Information found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "PII.0.3: Redact all PII"
|
|
when
|
|
$pii: TextEntity(type() == "PII", dictionaryEntry)
|
|
then
|
|
$pii.redact("PII.0.3", "Personal Information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
|
|
end
|
|
|
|
|
|
// Rule unit: PII.1
|
|
rule "PII.1.0: Redact Emails by RegEx"
|
|
when
|
|
$section: Section(containsString("@"))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
|
.forEach(emailEntity -> emailEntity.redact("PII.1.0", "Found by Email Regex", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.1.1: Redact Emails by RegEx (Non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("@"))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
|
.forEach(emailEntity -> emailEntity.redact("PII.1.1", "Found by Email Regex", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.1.2: Redact Emails by RegEx (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsString("@"))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
|
.forEach(emailEntity -> emailEntity.redact("PII.1.2", "Found by Email Regex", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.1.5: Redact Emails by RegEx"
|
|
when
|
|
$section: Section(containsString("@"))
|
|
then
|
|
entityCreationService.byRegex("\\b([A-Za-z0-9._%+\\-]+@[A-Za-z0-9.\\-]+\\.[A-Za-z\\-]{1,23}[A-Za-z])\\b", "PII", EntityType.ENTITY, 1, $section)
|
|
.forEach(emailEntity -> emailEntity.redact("PII.1.5", "Found by Email Regex", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
rule "PII.1.6: Redact typoed Emails with indicator"
|
|
when
|
|
$section: Section(containsString("@") || containsStringIgnoreCase("mail"))
|
|
then
|
|
entityCreationService.byRegexIgnoreCase("mail[:\\.\\s]{1,2}([\\w\\/\\-\\{\\(\\. ]{3,20}(@|a|f)\\s?[\\w\\/\\-\\{\\(\\. ]{3,20}(\\. \\w{2,4}\\b|\\.\\B|\\.\\w{1,4}\\b))", "PII", EntityType.ENTITY, 1, $section)
|
|
.forEach(emailEntity -> emailEntity.redact("PII.1.6", "Personal information found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.4
|
|
rule "PII.4.0: Redact line after contact information keywords"
|
|
when
|
|
$contactKeyword: String() from List.of("Contact point:",
|
|
"Contact:",
|
|
"Alternative contact:",
|
|
"European contact:",
|
|
"No:",
|
|
"Contact:",
|
|
"Tel.:",
|
|
"Tel:",
|
|
"Telephone number:",
|
|
"Telephone No:",
|
|
"Telephone:",
|
|
"Phone No.",
|
|
"Phone:",
|
|
"Fax number:",
|
|
"Fax:",
|
|
"E-mail:",
|
|
"Email:",
|
|
"e-mail:",
|
|
"E-mail address:")
|
|
$section: Section(containsString($contactKeyword))
|
|
then
|
|
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.4.0", "Found after \"" + $contactKeyword + "\" contact keyword", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.4.1: Redact line after contact information keywords"
|
|
when
|
|
$contactKeyword: String() from List.of("Contact point:",
|
|
"Contact:",
|
|
"Alternative contact:",
|
|
"European contact:",
|
|
"No:",
|
|
"Contact:",
|
|
"Tel.:",
|
|
"Tel:",
|
|
"Telephone number:",
|
|
"Telephone No:",
|
|
"Telephone:",
|
|
"Phone No.",
|
|
"Phone:",
|
|
"Fax number:",
|
|
"Fax:",
|
|
"E-mail:",
|
|
"Email:",
|
|
"e-mail:",
|
|
"E-mail address:")
|
|
$section: Section(containsString($contactKeyword))
|
|
then
|
|
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.4.1", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
rule "PII.4.2: Redact line after contact information keywords (Non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$contactKeyword: String() from List.of("Contact point:",
|
|
"Contact:",
|
|
"Alternative contact:",
|
|
"European contact:",
|
|
"No:",
|
|
"Contact:",
|
|
"Tel.:",
|
|
"Tel:",
|
|
"Telephone number:",
|
|
"Telephone No:",
|
|
"Telephone:",
|
|
"Phone No.",
|
|
"Phone:",
|
|
"Fax number:",
|
|
"Fax:",
|
|
"E-mail:",
|
|
"Email:",
|
|
"e-mail:",
|
|
"E-mail address:")
|
|
$section: Section(containsString($contactKeyword))
|
|
then
|
|
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.4.2", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.4.3: Redact line after contact information keywords (Vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$contactKeyword: String() from List.of("Contact point:",
|
|
"Contact:",
|
|
"Alternative contact:",
|
|
"European contact:",
|
|
"No:",
|
|
"Contact:",
|
|
"Tel.:",
|
|
"Tel:",
|
|
"Telephone number:",
|
|
"Telephone No:",
|
|
"Telephone:",
|
|
"Phone No.",
|
|
"Phone:",
|
|
"Fax number:",
|
|
"Fax:",
|
|
"E-mail:",
|
|
"Email:",
|
|
"e-mail:",
|
|
"E-mail address:")
|
|
$section: Section(containsString($contactKeyword))
|
|
then
|
|
entityCreationService.lineAfterString($contactKeyword, "PII", EntityType.ENTITY, $section)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.4.3", "Found after \"" + $contactKeyword + "\" contact keyword", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.6
|
|
rule "PII.6.0: Redact line between contact keywords"
|
|
when
|
|
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.6.0", "Found between contact keywords", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.6.1: Redact line between contact keywords (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.6.1", "Found between contact keywords", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.6.2: Redact line between contact keywords (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.6.2", "Found between contact keywords", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.6.3: Redact line between contact keywords (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section((containsString("No:") && containsString("Fax")) || (containsString("Contact:") && containsString("Tel")))
|
|
then
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
)
|
|
.forEach(contactEntity -> contactEntity.redact("PII.6.3", "Found between contact keywords", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.7
|
|
rule "PII.7.0: Redact contact information if applicant is found"
|
|
when
|
|
$section: Section(getHeadline().containsString("applicant") ||
|
|
getHeadline().containsString("Primary contact") ||
|
|
getHeadline().containsString("Alternative contact") ||
|
|
containsString("Applicant") ||
|
|
containsString("Telephone number:"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.7.0", "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.7.1: Redact contact information if applicant is found (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(getHeadline().containsString("applicant") ||
|
|
getHeadline().containsString("Primary contact") ||
|
|
getHeadline().containsString("Alternative contact") ||
|
|
containsString("Applicant") ||
|
|
containsString("Telephone number:"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.7.1", "Applicant information was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.7.2: Redact contact information if applicant is found (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(getHeadline().containsString("applicant") ||
|
|
getHeadline().containsString("Primary contact") ||
|
|
getHeadline().containsString("Alternative contact") ||
|
|
containsString("Applicant") ||
|
|
containsString("Telephone number:"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.7.2", "Applicant information was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.8
|
|
rule "PII.8.0: Redact contact information if producer is found"
|
|
when
|
|
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
|
|
containsStringIgnoreCase("producer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer:") ||
|
|
containsStringIgnoreCase("Producer or producers of the active substance"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.8.0", "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.8.1: Redact contact information if producer is found (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
|
|
containsStringIgnoreCase("producer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer:") ||
|
|
containsStringIgnoreCase("Producer or producers of the active substance"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.8.1", "Producer was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.8.2: Redact contact information if producer is found (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$section: Section(containsStringIgnoreCase("producer of the plant protection") ||
|
|
containsStringIgnoreCase("producer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer of the active substance") ||
|
|
containsStringIgnoreCase("manufacturer:") ||
|
|
containsStringIgnoreCase("Producer or producers of the active substance"))
|
|
then
|
|
Stream.concat(entityCreationService.lineAfterStrings(List.of("Contact point:", "Contact:", "Alternative contact:", "European contact:", "No:", "Contact:", "Tel.:", "Tel:", "Telephone number:",
|
|
"Telephone No:", "Telephone:", "Phone No.", "Phone:", "Fax number:", "Fax:", "E-mail:", "Email:", "e-mail:", "E-mail address:"), "PII", EntityType.ENTITY, $section),
|
|
Stream.concat(
|
|
entityCreationService.betweenStrings("No:", "Fax", "PII", EntityType.ENTITY, $section),
|
|
entityCreationService.betweenStrings("Contact:", "Tel", "PII", EntityType.ENTITY, $section)
|
|
))
|
|
.forEach(entity -> entity.redact("PII.8.2", "Producer was found", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.9
|
|
rule "PII.9.0: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
|
|
when
|
|
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
|
|
then
|
|
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
|
|
.forEach(authorEntity -> authorEntity.redact("PII.9.0", "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)"));
|
|
end
|
|
|
|
rule "PII.9.3: Redact between \"AUTHOR(S)\" and \"(STUDY) COMPLETION DATE\""
|
|
when
|
|
$document: Document(containsStringIgnoreCase("AUTHOR(S)"), containsAnyStringIgnoreCase("COMPLETION DATE", "STUDY COMPLETION DATE"))
|
|
then
|
|
entityCreationService.shortestBetweenAnyStringIgnoreCase(List.of("AUTHOR(S)", "AUTHOR(S):"), List.of("COMPLETION DATE", "COMPLETION DATE:", "STUDY COMPLETION DATE", "STUDY COMPLETION DATE:"), "PII", EntityType.ENTITY, $document)
|
|
.forEach(authorEntity -> authorEntity.redact("PII.9.3", "AUTHOR(S) was found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.11
|
|
rule "PII.11.0: Redact On behalf of Sequani Ltd.:"
|
|
when
|
|
$section: Section(!hasTables(), containsString("On behalf of Sequani Ltd.: Name Title"))
|
|
then
|
|
entityCreationService.betweenStrings("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", EntityType.ENTITY, $section)
|
|
.forEach(authorEntity -> authorEntity.redact("PII.11.0", "On behalf of Sequani Ltd.: Name Title was found", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
// Rule unit: PII.12
|
|
rule "PII.12.0: Expand PII entities with salutation prefix"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
|
|
then
|
|
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
|
|
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.0", "Expanded PII with salutation prefix", "Article 39(e)(3) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
rule "PII.12.1: Expand PII entities with salutation prefix"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$entityToExpand: TextEntity(type() == "PII", anyMatch(textBefore, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*"))
|
|
then
|
|
entityCreationService.byPrefixExpansionRegex($entityToExpand, "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*")
|
|
.ifPresent(expandedEntity -> expandedEntity.apply("PII.12.1", "Expanded PII with salutation prefix", "Article 39(e)(2) of Regulation (EC) No 178/2002"));
|
|
end
|
|
|
|
|
|
//------------------------------------ Other rules ------------------------------------
|
|
|
|
// Rule unit: ETC.1
|
|
rule "ETC.1.0: Redact Purity"
|
|
when
|
|
$section: Section(containsStringIgnoreCase("purity"))
|
|
then
|
|
entityCreationService.byRegex("\\bPurity:\\s*(<?>?\\s*\\d{1,2}(?:\\.\\d{1,2})?\\s*%)", "purity", EntityType.ENTITY, 1, $section)
|
|
.forEach(entity -> entity.redact("ETC.1.0", "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)"));
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.2
|
|
rule "ETC.2.0: Redact signatures"
|
|
when
|
|
$signature: Image(imageType == ImageType.SIGNATURE)
|
|
then
|
|
$signature.redact("ETC.2.0", "Signature Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "ETC.2.1: Redact signatures (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$signature: Image(imageType == ImageType.SIGNATURE)
|
|
then
|
|
$signature.redact("ETC.2.1", "Signature Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "ETC.2.2: Redact signatures (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$signature: Image(imageType == ImageType.SIGNATURE)
|
|
then
|
|
$signature.redact("ETC.2.2", "Signature Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "ETC.2.3: Redact signatures"
|
|
when
|
|
$signature: Image(imageType == ImageType.SIGNATURE)
|
|
then
|
|
$signature.redact("ETC.2.3", "Signature Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.3
|
|
rule "ETC.3.0: Redact logos"
|
|
when
|
|
$logo: Image(imageType == ImageType.LOGO)
|
|
then
|
|
$logo.redact("ETC.3.0", "Logo Found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
|
end
|
|
|
|
rule "ETC.3.1: Skip logos (non vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$logo: Image(imageType == ImageType.LOGO)
|
|
then
|
|
$logo.skip("ETC.3.1", "Logo Found");
|
|
end
|
|
|
|
rule "ETC.3.2: Redact logos (vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$logo: Image(imageType == ImageType.LOGO)
|
|
then
|
|
$logo.redact("ETC.3.2", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "ETC.3.3: Redact logos"
|
|
when
|
|
$logo: Image(imageType == ImageType.LOGO)
|
|
then
|
|
$logo.redact("ETC.3.3", "Logo Found", "Article 4(1)(b), Regulation (EC) No 1049/2001 (Personal data)");
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.4
|
|
rule "ETC.4.0: Redact dossier dictionary entries"
|
|
when
|
|
$dossierRedaction: TextEntity(type() == "dossier_redaction")
|
|
then
|
|
$dossierRedaction.redact("ETC.4.0", "Specification of impurity found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.5
|
|
rule "ETC.5.0: Skip dossier_redaction entries if confidentiality is 'confidential'"
|
|
when
|
|
FileAttribute(label == "Confidentiality", value == "confidential")
|
|
$dossierRedaction: TextEntity(type() == "dossier_redaction")
|
|
then
|
|
$dossierRedaction.skip("ETC.5.0", "Ignore dossier_redaction when confidential");
|
|
$dossierRedaction.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "ETC.5.1: Remove dossier_redaction entries if confidentiality is not 'confidential'"
|
|
salience 256
|
|
when
|
|
not FileAttribute(label == "Confidentiality", value == "confidential")
|
|
$dossierRedaction: TextEntity(type() == "dossier_redaction")
|
|
then
|
|
$dossierRedaction.remove("ETC.5.1", "Remove dossier_redaction when not confidential");
|
|
retract($dossierRedaction);
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.6
|
|
rule "ETC.6.0: Redact CAS Number"
|
|
when
|
|
$table: Table(hasHeader("Sample #"))
|
|
then
|
|
$table.streamTableCellsWithHeader("Sample #")
|
|
.map(tableCell -> entityCreationService.bySemanticNode(tableCell, "PII", EntityType.ENTITY))
|
|
.filter(Optional::isPresent)
|
|
.map(Optional::get)
|
|
.forEach(redactionEntity -> redactionEntity.redact("ETC.6.0", "Sample # found in Header", "Reg (EC) No 1107/2009 Art. 63 (2g)"));
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.7
|
|
rule "ETC.7.0: Guidelines FileAttributes"
|
|
when
|
|
$section: Section(!hasTables(), containsAnyString("DATA REQUIREMENT(S):", "TEST GUIDELINE(S):") && containsAnyString("OECD", "EPA", "OPPTS"))
|
|
then
|
|
RedactionSearchUtility.findTextRangesByRegex("OECD (No\\.? )?\\d{3}( \\(\\d{4}\\))?", $section.getTextBlock()).stream()
|
|
.map(boundary -> $section.getTextBlock().subSequence(boundary).toString())
|
|
.map(value -> FileAttribute.builder().label("OECD Number").value(value).build())
|
|
.forEach(fileAttribute -> insert(fileAttribute));
|
|
end
|
|
|
|
|
|
// Rule unit: ETC.8
|
|
rule "ETC.8.0: Redact formulas (vertebrate study)"
|
|
when
|
|
not FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$logo: Image(imageType == ImageType.FORMULA)
|
|
then
|
|
$logo.redact("ETC.8.0", "Logo Found", "Article 39(e)(3) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
rule "ETC.8.1: Redact formulas (non vertebrate study)"
|
|
when
|
|
FileAttribute(label == "Vertebrate Study", value soundslike "Yes" || value.toLowerCase() == "y")
|
|
$logo: Image(imageType == ImageType.FORMULA)
|
|
then
|
|
$logo.redact("ETC.8.1", "Logo Found", "Article 39(e)(2) of Regulation (EC) No 178/2002");
|
|
end
|
|
|
|
|
|
//------------------------------------ AI rules ------------------------------------
|
|
|
|
// Rule unit: AI.0
|
|
rule "AI.0.0: Add all NER Entities of type CBI_author"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("CBI_author"))
|
|
then
|
|
nerEntities.streamEntitiesOfType("CBI_author")
|
|
.forEach(nerEntity -> entityCreationService.optionalByNerEntity(nerEntity, EntityType.RECOMMENDATION, document));
|
|
end
|
|
|
|
|
|
// Rule unit: AI.1
|
|
rule "AI.1.0: Combine and add NER Entities as CBI_address"
|
|
salience 999
|
|
when
|
|
nerEntities: NerEntities(hasEntitiesOfType("ORG") || hasEntitiesOfType("STREET") || hasEntitiesOfType("CITY"))
|
|
then
|
|
entityCreationService.combineNerEntitiesToCbiAddressDefaults(nerEntities, "CBI_address", EntityType.RECOMMENDATION, document).toList();
|
|
end
|
|
|
|
|
|
//------------------------------------ Manual changes rules ------------------------------------
|
|
|
|
// Rule unit: MAN.0
|
|
rule "MAN.0.0: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$entityToBeResized: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
manualChangesApplicationService.resize($entityToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
update($entityToBeResized);
|
|
$entityToBeResized.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "MAN.0.1: Apply manual resize redaction"
|
|
salience 128
|
|
when
|
|
$resizeRedaction: ManualResizeRedaction($id: annotationId, $requestDate: requestDate)
|
|
not ManualResizeRedaction(annotationId == $id, requestDate.isBefore($requestDate))
|
|
$imageToBeResized: Image(id == $id)
|
|
then
|
|
manualChangesApplicationService.resizeImage($imageToBeResized, $resizeRedaction);
|
|
retract($resizeRedaction);
|
|
update($imageToBeResized);
|
|
update($imageToBeResized.getParent());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.1
|
|
rule "MAN.1.0: Apply id removals that are valid and not in forced redactions to Entity"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId, !removeFromDictionary, !removeFromAllDossiers)
|
|
$entityToBeRemoved: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
|
update($entityToBeRemoved);
|
|
retract($idRemoval);
|
|
$entityToBeRemoved.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
rule "MAN.1.1: Apply id removals that are valid and not in forced redactions to Image"
|
|
salience 128
|
|
when
|
|
$idRemoval: IdRemoval($id: annotationId)
|
|
$imageEntityToBeRemoved: Image($id == id)
|
|
then
|
|
$imageEntityToBeRemoved.getManualOverwrite().addChange($idRemoval);
|
|
update($imageEntityToBeRemoved);
|
|
retract($idRemoval);
|
|
update($imageEntityToBeRemoved.getParent());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.2
|
|
rule "MAN.2.0: Apply force redaction"
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId)
|
|
$entityToForce: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToForce.getManualOverwrite().addChange($force);
|
|
update($entityToForce);
|
|
$entityToForce.getIntersectingNodes().forEach(node -> update(node));
|
|
retract($force);
|
|
end
|
|
|
|
rule "MAN.2.1: Apply force redaction to images"
|
|
salience 128
|
|
when
|
|
$force: ManualForceRedaction($id: annotationId)
|
|
$imageToForce: Image(id == $id)
|
|
then
|
|
$imageToForce.getManualOverwrite().addChange($force);
|
|
update($imageToForce);
|
|
update($imageToForce.getParent());
|
|
retract($force);
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.3
|
|
rule "MAN.3.0: Apply entity recategorization"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() != $type)
|
|
then
|
|
$entityToBeRecategorized.getIntersectingNodes().forEach(node -> update(node));
|
|
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
|
|
update($entityToBeRecategorized);
|
|
retract($recategorization);
|
|
end
|
|
|
|
rule "MAN.3.1: Apply entity recategorization of same type"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, $type: type, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$entityToBeRecategorized: TextEntity(matchesAnnotationId($id), type() == $type)
|
|
then
|
|
$entityToBeRecategorized.getManualOverwrite().addChange($recategorization);
|
|
retract($recategorization);
|
|
end
|
|
|
|
rule "MAN.3.2: Apply image recategorization"
|
|
salience 128
|
|
when
|
|
$recategorization: ManualRecategorization($id: annotationId, $requestDate: requestDate)
|
|
not ManualRecategorization($id == annotationId, requestDate.isBefore($requestDate))
|
|
$imageToBeRecategorized: Image($id == id)
|
|
then
|
|
manualChangesApplicationService.recategorize($imageToBeRecategorized, $recategorization);
|
|
update($imageToBeRecategorized);
|
|
update($imageToBeRecategorized.getParent());
|
|
retract($recategorization);
|
|
end
|
|
|
|
rule "MAN.3.3: Apply recategorization entities by default"
|
|
salience 128
|
|
when
|
|
$entity: TextEntity(getManualOverwrite().getRecategorized().orElse(false), !dictionary.isHint(type()))
|
|
then
|
|
$entity.apply("MAN.3.3", "Recategorized entities are applied by default.", $entity.legalBasis());
|
|
end
|
|
|
|
|
|
// Rule unit: MAN.4
|
|
rule "MAN.4.0: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
|
|
$imageToBeRecategorized: Image($id == id)
|
|
then
|
|
$imageToBeRecategorized.getManualOverwrite().addChange($legalBasisChange);
|
|
update($imageToBeRecategorized)
|
|
retract($legalBasisChange)
|
|
end
|
|
|
|
rule "MAN.4.1: Apply legal basis change"
|
|
salience 128
|
|
when
|
|
$legalBasisChange: ManualLegalBasisChange($id: annotationId)
|
|
$entityToBeChanged: TextEntity(matchesAnnotationId($id))
|
|
then
|
|
$entityToBeChanged.getManualOverwrite().addChange($legalBasisChange);
|
|
update($entityToBeChanged)
|
|
retract($legalBasisChange)
|
|
end
|
|
|
|
|
|
//------------------------------------ Entity merging rules ------------------------------------
|
|
|
|
// Rule unit: X.0
|
|
rule "X.0.0: Remove Entity contained by Entity of same type"
|
|
salience 65
|
|
when
|
|
$larger: TextEntity($type: type(), $entityType: entityType, active())
|
|
$contained: TextEntity(containedBy($larger), type() == $type, entityType == $entityType, this != $larger, !hasManualChanges())
|
|
then
|
|
$contained.remove("X.0.0", "remove Entity contained by Entity of same type");
|
|
retract($contained);
|
|
end
|
|
|
|
|
|
// Rule unit: X.1
|
|
rule "X.1.0: Merge intersecting Entities of same type"
|
|
salience 64
|
|
when
|
|
$first: TextEntity($type: type(), $entityType: entityType, !resized(), active())
|
|
$second: TextEntity(intersects($first), type() == $type, entityType == $entityType, this != $first, !hasManualChanges(), active())
|
|
then
|
|
TextEntity mergedEntity = entityCreationService.mergeEntitiesOfSameType(List.of($first, $second), $type, $entityType, document);
|
|
$first.remove("X.1.0", "merge intersecting Entities of same type");
|
|
$second.remove("X.1.0", "merge intersecting Entities of same type");
|
|
retract($first);
|
|
retract($second);
|
|
mergedEntity.getIntersectingNodes().forEach(node -> update(node));
|
|
end
|
|
|
|
|
|
// Rule unit: X.2
|
|
rule "X.2.0: Remove Entity of type ENTITY when contained by FALSE_POSITIVE"
|
|
salience 64
|
|
when
|
|
$falsePositive: TextEntity($type: type(), entityType == EntityType.FALSE_POSITIVE, active())
|
|
$entity: TextEntity(containedBy($falsePositive), type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
|
|
then
|
|
$entity.getIntersectingNodes().forEach(node -> update(node));
|
|
$entity.remove("X.2.0", "remove Entity of type ENTITY when contained by FALSE_POSITIVE");
|
|
retract($entity)
|
|
end
|
|
|
|
|
|
// Rule unit: X.3
|
|
rule "X.3.0: Remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION"
|
|
salience 64
|
|
when
|
|
$falseRecommendation: TextEntity($type: type(), entityType == EntityType.FALSE_RECOMMENDATION, active())
|
|
$recommendation: TextEntity(containedBy($falseRecommendation), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
|
|
then
|
|
$recommendation.remove("X.3.0", "remove Entity of type RECOMMENDATION when contained by FALSE_RECOMMENDATION");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.4
|
|
rule "X.4.0: Remove Entity of type RECOMMENDATION when text range equals ENTITY with same type"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
|
$recommendation: TextEntity(getTextRange().equals($entity.getTextRange()), type() == $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
|
|
then
|
|
$entity.addEngines($recommendation.getEngines());
|
|
$recommendation.remove("X.4.0", "remove Entity of type RECOMMENDATION when text range equals ENTITY with same type");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.5
|
|
rule "X.5.0: Remove Entity of type RECOMMENDATION when intersected by ENTITY"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity((entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
|
$recommendation: TextEntity(intersects($entity), entityType == EntityType.RECOMMENDATION, !hasManualChanges())
|
|
then
|
|
$recommendation.remove("X.5.0", "remove Entity of type RECOMMENDATION when intersected by ENTITY");
|
|
retract($recommendation);
|
|
end
|
|
|
|
rule "X.5.1: Remove Entity of type RECOMMENDATION when contained by RECOMMENDATION"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity($type: type(), entityType == EntityType.RECOMMENDATION, active())
|
|
$recommendation: TextEntity(containedBy($entity), type() != $type, entityType == EntityType.RECOMMENDATION, !hasManualChanges())
|
|
then
|
|
$recommendation.remove("X.5.1", "remove Entity of type RECOMMENDATION when contained by RECOMMENDATION");
|
|
retract($recommendation);
|
|
end
|
|
|
|
|
|
// Rule unit: X.6
|
|
rule "X.6.0: Remove Entity of lower rank, when contained by entity of type ENTITY or HINT"
|
|
salience 32
|
|
when
|
|
$higherRank: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
|
$lowerRank: TextEntity(containedBy($higherRank), type() != $type, dictionary.getDictionaryRank(type) < dictionary.getDictionaryRank($type), !hasManualChanges())
|
|
then
|
|
$lowerRank.getIntersectingNodes().forEach(node -> update(node));
|
|
$lowerRank.remove("X.6.0", "remove Entity of lower rank, when contained by entity of type ENTITY or HINT");
|
|
retract($lowerRank);
|
|
end
|
|
|
|
rule "X.6.1: remove Entity, when contained in another entity of type ENTITY or HINT with larger text range"
|
|
salience 32
|
|
when
|
|
$outer: TextEntity($type: type(), (entityType == EntityType.ENTITY || entityType == EntityType.HINT), active())
|
|
$inner: TextEntity(containedBy($outer), type() != $type, $outer.getTextRange().length > getTextRange().length(), !hasManualChanges())
|
|
then
|
|
$inner.getIntersectingNodes().forEach(node -> update(node));
|
|
$inner.remove("X.6.1", "remove Entity, when contained in another entity of type ENTITY or HINT with larger text range");
|
|
retract($inner);
|
|
end
|
|
|
|
|
|
// Rule unit: X.8
|
|
rule "X.8.0: Remove Entity when text range and type equals to imported Entity"
|
|
salience 257
|
|
when
|
|
$entity: TextEntity($type: type(), engines contains Engine.IMPORTED, active())
|
|
$other: TextEntity(getTextRange().equals($entity.getTextRange()), this != $entity, type() == $type, engines not contains Engine.IMPORTED)
|
|
then
|
|
$other.remove("X.8.0", "remove Entity when text range and type equals to imported Entity");
|
|
$entity.addEngines($other.getEngines());
|
|
retract($other);
|
|
end
|
|
|
|
rule "X.8.1: Remove Entity when intersected by imported Entity"
|
|
salience 256
|
|
when
|
|
$entity: TextEntity(engines contains Engine.IMPORTED, active())
|
|
$other: TextEntity(intersects($entity), this != $entity, engines not contains Engine.IMPORTED)
|
|
then
|
|
$other.remove("X.8.1", "remove Entity when intersected by imported Entity");
|
|
retract($other);
|
|
end
|
|
|
|
|
|
// Rule unit: X.9
|
|
rule "X.9.0: Merge mostly contained signatures"
|
|
when
|
|
$aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI)
|
|
$signature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.ALGORITHM, mostlyContains($aiSignature, 0.8))
|
|
then
|
|
$aiSignature.remove("X.9.0", "removed because already contained by alogrithm signature");
|
|
$signature.addEngine(LayoutEngine.AI);
|
|
end
|
|
|
|
|
|
// Rule unit: X.10
|
|
rule "X.10.0: remove false positives of ai"
|
|
when
|
|
$anyImage: Image(engines contains LayoutEngine.ALGORITHM)
|
|
$aiSignature: Image(imageType == ImageType.SIGNATURE, engines contains LayoutEngine.AI, !mostlyContainedBy($anyImage, 0.8))
|
|
then
|
|
$aiSignature.remove("X.10.0", "Removed because false positive");
|
|
end
|
|
|
|
|
|
// Rule unit: X.11
|
|
rule "X.11.0: Remove dictionary entity which intersects with a manual entity"
|
|
salience 64
|
|
when
|
|
$manualEntity: TextEntity(engines contains Engine.MANUAL, active())
|
|
$dictionaryEntity: TextEntity(intersects($manualEntity), dictionaryEntry, engines not contains Engine.MANUAL)
|
|
then
|
|
$dictionaryEntity.remove("X.11.0", "remove dictionary entity which intersects with a manual entity");
|
|
retract($dictionaryEntity);
|
|
end
|
|
|
|
|
|
//------------------------------------ Dictionary merging rules ------------------------------------
|
|
|
|
// Rule unit: DICT.0
|
|
rule "DICT.0.0: Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL"
|
|
salience 64
|
|
when
|
|
$dictionaryRemoval: TextEntity($type: type(), entityType == EntityType.DICTIONARY_REMOVAL, engines contains Engine.DOSSIER_DICTIONARY, active())
|
|
$entity: TextEntity(getTextRange().equals($dictionaryRemoval.getTextRange()), engines contains Engine.DICTIONARY, type() == $type, (entityType == EntityType.ENTITY || entityType == EntityType.HINT), !hasManualChanges())
|
|
then
|
|
$entity.getIntersectingNodes().forEach(node -> update(node));
|
|
$entity.ignore("DICT.0.0", "Ignore Template Dictionary Entity when contained by Dossier Dictionary DICTIONARY_REMOVAL");
|
|
end
|
|
|
|
|
|
//------------------------------------ File attributes rules ------------------------------------
|
|
|
|
// Rule unit: FA.1
|
|
rule "FA.1.0: Remove duplicate FileAttributes"
|
|
salience 64
|
|
when
|
|
$fileAttribute: FileAttribute($label: label, $value: value)
|
|
$duplicate: FileAttribute(this != $fileAttribute, label == $label, value == $value)
|
|
then
|
|
retract($duplicate);
|
|
end
|
|
|
|
|
|
//------------------------------------ Local dictionary search rules ------------------------------------
|
|
|
|
// Rule unit: LDS.0
|
|
rule "LDS.0.0: Run local dictionary search"
|
|
agenda-group "LOCAL_DICTIONARY_ADDS"
|
|
salience -999
|
|
when
|
|
$dictionaryModel: DictionaryModel(!localEntriesWithMatchedRules.isEmpty()) from dictionary.getDictionaryModels()
|
|
then
|
|
entityCreationService.bySearchImplementation($dictionaryModel.getLocalSearch(), $dictionaryModel.getType(), EntityType.RECOMMENDATION, document)
|
|
.forEach(entity -> {
|
|
Collection<MatchedRule> matchedRules = $dictionaryModel.getMatchedRulesForLocalDictionaryEntry(entity.getValue());
|
|
matchedRules.forEach(matchedRule -> entity.addMatchedRule(matchedRule.asSkippedIfApplied()));
|
|
});
|
|
end
|