RSS-83: Test new rules for headline detection
This commit is contained in:
deiflaender
parent
074205aa4d
commit
fb1892ea20
@ -45,6 +45,7 @@ public class BlockificationService {
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
|
||||
boolean lineSeparation2 = prev != null && minY - word.getY2() > word.getHeight() * 0.75 && word.getFontSize() != prev.getFontSize();
|
||||
boolean startFromTop = word.getY1() > maxY + word.getHeight();
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
|
||||
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
|
||||
@ -53,7 +54,9 @@ public class BlockificationService {
|
||||
.getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
|
||||
.getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
|
||||
boolean isOtherFont = prev != null && word.getFontStyle() != prev.getFontStyle();
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling || lineSeparation2 || isOtherFont)) {
|
||||
|
||||
Orientation prevOrientation = null;
|
||||
if (!chunkBlockList1.isEmpty()) {
|
||||
|
||||
@ -1,9 +1,11 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -11,6 +13,8 @@ import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@ -51,10 +55,18 @@ public class ClassificationService {
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
textBlock.setClassification("Other");
|
||||
return;
|
||||
}
|
||||
var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{0,2}\\s[A-Za-z]{3,50}", true);
|
||||
var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
|
||||
|
||||
Matcher matcher = pattern.matcher(textBlock.toString());
|
||||
Matcher matcher2 = pattern2.matcher(textBlock.toString());
|
||||
|
||||
|
||||
|
||||
// if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// textBlock.setClassification("Other");
|
||||
// return;
|
||||
// }
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
@ -64,55 +76,54 @@ public class ClassificationService {
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
// } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
// .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
// .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
// .getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
// if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
// textBlock.setClassification("Title");
|
||||
// }
|
||||
} else
|
||||
if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 2.9
|
||||
&& !textBlock.getOrientation().equals(Orientation.RIGHT)
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
textBlock.setClassification("H " + i);
|
||||
&& (textBlock.getMostPopularWordStyle().contains("bold") || textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT)))) {
|
||||
textBlock.setClassification("H 1");
|
||||
document.setHeadlines(true);
|
||||
|
||||
} else
|
||||
if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
|
||||
textBlock.setClassification("H 2");
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
// else if (textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
|
||||
// .getMostPopular()
|
||||
// .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
// textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
// document.setHeadlines(true);
|
||||
// }
|
||||
// else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
// .getFontSizeCounter()
|
||||
// .getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
// .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
// textBlock.setClassification("TextBlock Bold");
|
||||
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
// .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
// .equals(document.getFontStyleCounter()
|
||||
// .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
// .getMostPopular()) {
|
||||
// textBlock.setClassification("TextBlock");
|
||||
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
// .getFontSizeCounter()
|
||||
// .getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
// .equals("italic") && !document.getFontStyleCounter()
|
||||
// .getMostPopular()
|
||||
// .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
// textBlock.setClassification("TextBlock Italic");
|
||||
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
// textBlock.setClassification("TextBlock Unknown");
|
||||
else {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
} else {
|
||||
textBlock.setClassification("Other");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -68,7 +68,7 @@ public class SectionsBuilderService {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if (document.isHeadlines()) {
|
||||
|
||||
@ -334,7 +334,9 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void titleExtraction() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
|
||||
System.out.println("Detailed quantitative and qualitative information on the composition of the plant ".length());
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Trinexapac/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf");
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
@ -1094,7 +1096,7 @@ public class RedactionIntegrationTest {
|
||||
public void classificationTest() throws IOException {
|
||||
|
||||
System.out.println("classificationTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Trinexapac/91 Trinexapac-ethyl_RAR_01_Volume_1_2018-02-23.pdf");
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
|
||||
@ -15,379 +15,9 @@ global Section section
|
||||
// end
|
||||
|
||||
|
||||
rule "0: Add CBI_author from ai"
|
||||
rule "1: Find headlines "
|
||||
when
|
||||
Section(aiMatchesType("CBI_author"))
|
||||
Section(text.length() > 1)
|
||||
then
|
||||
section.addAiEntities("CBI_author", "CBI_author");
|
||||
end
|
||||
|
||||
rule "0: Combine ai types CBI_author from ai"
|
||||
when
|
||||
Section(aiMatchesType("ORG"))
|
||||
then
|
||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Expand CBI Authors with firstname initials"
|
||||
when
|
||||
Section(matchesType("CBI_author"))
|
||||
then
|
||||
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
|
||||
end
|
||||
|
||||
rule "0: Expand CBI_author and PII matches with salutation prefix"
|
||||
when
|
||||
Section((matchesType("CBI_author") || matchesType("PII")) && (
|
||||
searchText.contains("Mr")
|
||||
|| searchText.contains("Mrs")
|
||||
|| searchText.contains("Ms")
|
||||
|| searchText.contains("Miss")
|
||||
|| searchText.contains("Sir")
|
||||
|| searchText.contains("Madam")
|
||||
|| searchText.contains("Madame")
|
||||
|| searchText.contains("Mme")
|
||||
))
|
||||
then
|
||||
section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
||||
section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
||||
end
|
||||
|
||||
|
||||
rule "1: Redacted because Section contains Vertebrate"
|
||||
when
|
||||
Section(matchesType("vertebrate"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "2: Not Redacted because Section contains no Vertebrate"
|
||||
when
|
||||
Section(!matchesType("vertebrate"))
|
||||
then
|
||||
section.redactNot("CBI_author", 2, "No Vertebrate found");
|
||||
section.redactNot("CBI_address", 2, "No Vertebrate found");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
||||
then
|
||||
section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
|
||||
section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
|
||||
end
|
||||
|
||||
|
||||
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
||||
then
|
||||
section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("published_information"))
|
||||
then
|
||||
section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
|
||||
section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
|
||||
end
|
||||
|
||||
|
||||
rule "6: Not redacted because Vertebrate Study = N"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
|
||||
then
|
||||
section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
|
||||
section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
|
||||
section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
|
||||
section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
|
||||
end
|
||||
|
||||
|
||||
rule "7: Redact if must redact entry is found"
|
||||
when
|
||||
Section(matchesType("must_redact"))
|
||||
then
|
||||
section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
|
||||
then
|
||||
section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "9: Redact sponsor company"
|
||||
when
|
||||
Section(searchText.toLowerCase().contains("batches produced at"))
|
||||
then
|
||||
section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("batches produced at", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "10: Redact determination of residues"
|
||||
when
|
||||
Section((
|
||||
searchText.toLowerCase.contains("determination of residues") ||
|
||||
searchText.toLowerCase.contains("determination of total residues")
|
||||
) && (
|
||||
searchText.toLowerCase.contains("livestock") ||
|
||||
searchText.toLowerCase.contains("live stock") ||
|
||||
searchText.toLowerCase.contains("tissue") ||
|
||||
searchText.toLowerCase.contains("tissues") ||
|
||||
searchText.toLowerCase.contains("liver") ||
|
||||
searchText.toLowerCase.contains("muscle") ||
|
||||
searchText.toLowerCase.contains("bovine") ||
|
||||
searchText.toLowerCase.contains("ruminant") ||
|
||||
searchText.toLowerCase.contains("ruminants")
|
||||
))
|
||||
then
|
||||
section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("determination of residues", "must_redact");
|
||||
section.addHintAnnotation("livestock", "must_redact");
|
||||
section.addHintAnnotation("live stock", "must_redact");
|
||||
section.addHintAnnotation("tissue", "must_redact");
|
||||
section.addHintAnnotation("tissues", "must_redact");
|
||||
section.addHintAnnotation("liver", "must_redact");
|
||||
section.addHintAnnotation("muscle", "must_redact");
|
||||
section.addHintAnnotation("bovine", "must_redact");
|
||||
section.addHintAnnotation("ruminant", "must_redact");
|
||||
section.addHintAnnotation("ruminants", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "11: Redact if CTL/* or BL/* was found"
|
||||
when
|
||||
Section(searchText.contains("CTL/") || searchText.contains("BL/"))
|
||||
then
|
||||
section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("CTL", "must_redact");
|
||||
section.addHintAnnotation("BL", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "12: Redact and add recommendation for et al. author"
|
||||
when
|
||||
Section(searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "13: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
Section(searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
|
||||
rule "14: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
Section(searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
end
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "14: Redacted PII Personal Identification Information"
|
||||
when
|
||||
Section(matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "15: Redact Emails by RegEx"
|
||||
when
|
||||
Section(searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Redact contact information"
|
||||
when
|
||||
Section(text.contains("Contact point:")
|
||||
|| text.contains("Phone:")
|
||||
|| text.contains("Fax:")
|
||||
|| text.contains("Tel.:")
|
||||
|| text.contains("Tel:")
|
||||
|| text.contains("E-mail:")
|
||||
|| text.contains("Email:")
|
||||
|| text.contains("e-mail:")
|
||||
|| text.contains("E-mail address:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| text.contains("Telephone number:")
|
||||
|| text.contains("Telephone No:")
|
||||
|| text.contains("Fax number:")
|
||||
|| text.contains("Telephone:")
|
||||
|| text.contains("European contact:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "17: Redact contact information if applicant is found"
|
||||
when
|
||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "18: Redact contact information if Producer is found"
|
||||
when
|
||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
||||
then
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "19: Redact AUTHOR(S)"
|
||||
when
|
||||
Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "20: Redact PERFORMING LABORATORY"
|
||||
when
|
||||
Section(searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact On behalf of Sequani Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "22: Redact On behalf of Syngenta Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "25: Redact Purity"
|
||||
when
|
||||
Section(searchText.contains("purity"))
|
||||
then
|
||||
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
|
||||
end
|
||||
|
||||
|
||||
rule "26: Redact signatures"
|
||||
when
|
||||
Section(matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "27: Redact formula"
|
||||
when
|
||||
Section(matchesImageType("formula"))
|
||||
then
|
||||
section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "28: Redact Logos"
|
||||
when
|
||||
Section(matchesImageType("logo"))
|
||||
then
|
||||
section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "29: Redact Dossier Redactions"
|
||||
when
|
||||
Section(matchesType("dossier_redactions"))
|
||||
then
|
||||
section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "30: Ignore dossier_redactions if confidential"
|
||||
when
|
||||
Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
|
||||
then
|
||||
section.ignore("dossier_redactions");
|
||||
end
|
||||
|
||||
// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
|
||||
// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
|
||||
rule "101: Redact CAS numbers"
|
||||
when
|
||||
Section(hasTableHeader("Sample #"))
|
||||
then
|
||||
section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redactHeadline("PII", 1, "Headline found", "n-a.");
|
||||
end
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user