Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
d7d2bfff6b | ||
|
|
65daab4372 | ||
|
|
5503c7929b | ||
|
|
7dae9e8f0b | ||
|
|
fb1892ea20 |
@ -45,6 +45,7 @@ public class BlockificationService {
|
||||
for (TextPositionSequence word : textPositions) {
|
||||
|
||||
boolean lineSeparation = minY - word.getY2() > word.getHeight() * 1.25;
|
||||
boolean lineSeparation2 = prev != null && minY - word.getY2() > word.getHeight() * 0.75 && word.getFontSize() != prev.getFontSize();
|
||||
boolean startFromTop = word.getY1() > maxY + word.getHeight();
|
||||
boolean splitByX = prev != null && maxX + 50 < word.getX1() && prev.getY1() == word.getY1();
|
||||
boolean newLineAfterSplit = prev != null && word.getY1() != prev.getY1() && wasSplitted && splitX1 != word.getX1();
|
||||
@ -53,7 +54,9 @@ public class BlockificationService {
|
||||
.getRotation() == 90 && isSplittedByRuling(maxX, minY, word.getX1(), word.getY1(), horizontalRulingLines) || word
|
||||
.getRotation() == 90 && isSplittedByRuling(minX, minY, word.getX1(), word.getY2(), verticalRulingLines);
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling)) {
|
||||
boolean isOtherFont = prev != null && word.getFontStyle() != prev.getFontStyle();
|
||||
|
||||
if (prev != null && (lineSeparation || startFromTop || splitByX || newLineAfterSplit || splittedByRuling || lineSeparation2 || isOtherFont)) {
|
||||
|
||||
Orientation prevOrientation = null;
|
||||
if (!chunkBlockList1.isEmpty()) {
|
||||
|
||||
@ -1,16 +1,23 @@
|
||||
package com.iqser.red.service.redaction.v1.server.classification.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Orientation;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.utils.PositionUtils;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Rectangle;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@Slf4j
|
||||
@ -32,12 +39,49 @@ public class ClassificationService {
|
||||
|
||||
for (Page page : document.getPages()) {
|
||||
Rectangle btf = page.isLandscape() ? landscapeBodyTextFrame : bodyTextFrame;
|
||||
|
||||
// if (page.getRotation() == 180 || page.getRotation() == 270){
|
||||
// var nBtf = getRectangle(btf.x, btf.y, btf.width, btf.height, page.getRotation(),((TextBlock) page.getTextBlocks().get(0)).getSequences().get(0).getPageWidth(),((TextBlock) page.getTextBlocks().get(0)).getSequences().get(0).getPageHeight());
|
||||
// btf = new Rectangle(nBtf.getTopLeft().getX(), nBtf.getTopLeft().getY(), nBtf.getWidth(), nBtf.getHeight());
|
||||
// }
|
||||
|
||||
page.setBodyTextFrame(btf);
|
||||
classifyPage(btf, page, document, headlineFontSizes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public com.iqser.red.service.redaction.v1.model.Rectangle getRectangle(float x, float y, float width, float height, int rotation, float pageWidth, float pageHeight) {
|
||||
|
||||
|
||||
Point2D bottomLeft = new Point2D.Double(x, pageHeight - y);
|
||||
Point2D topRight = new Point2D.Double(x + width,pageHeight - y - height);
|
||||
|
||||
AffineTransform transform = new AffineTransform();
|
||||
if (rotation == 0 || rotation == 180) {
|
||||
transform.rotate(Math.toRadians(rotation), pageWidth / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageHeight);
|
||||
transform.scale(1., -1.);
|
||||
} else if (rotation == 90) {
|
||||
transform.rotate(Math.toRadians(rotation), pageWidth / 2f, pageWidth / 2f);
|
||||
transform.translate(0f, pageWidth);
|
||||
transform.scale(1., -1.);
|
||||
} else {
|
||||
transform.rotate(Math.toRadians(rotation), pageHeight / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageWidth);
|
||||
transform.scale(1., -1.);
|
||||
}
|
||||
|
||||
bottomLeft = transform.transform(bottomLeft, null);
|
||||
topRight = transform.transform(topRight, null);
|
||||
|
||||
return new com.iqser.red.service.redaction.v1.model.Rectangle( //
|
||||
new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void classifyPage(Rectangle bodyTextFrame, Page page, Document document, List<Float> headlineFontSizes) {
|
||||
|
||||
for (AbstractTextContainer textBlock : page.getTextBlocks()) {
|
||||
@ -51,11 +95,44 @@ public class ClassificationService {
|
||||
public void classifyBlock(TextBlock textBlock, Rectangle bodyTextFrame, Page page, Document document,
|
||||
List<Float> headlineFontSizes) {
|
||||
|
||||
if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
textBlock.setClassification("Other");
|
||||
return;
|
||||
var pattern = Patterns.getCompiledPattern("^(\\d{1,2}\\.){1,3}\\d{1,2}\\s[A-Za-z]{3,50}", true);
|
||||
var pattern2 = Patterns.getCompiledPattern(".*\\d$", true);
|
||||
|
||||
Matcher matcher = pattern.matcher(textBlock.toString());
|
||||
Matcher matcher2 = pattern2.matcher(textBlock.toString());
|
||||
|
||||
|
||||
|
||||
// if (document.getFontSizeCounter().getMostPopular() == null) {
|
||||
// textBlock.setClassification("Other");
|
||||
// return;
|
||||
// }
|
||||
|
||||
// } else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
// .getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
// .getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
// .getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
// if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
// textBlock.setClassification("Title");
|
||||
// }
|
||||
|
||||
if (textBlock.getText().length() > 5 && textBlock.getMostPopularWordHeight() > document.getTextHeightCounter().getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 5.9
|
||||
&& !textBlock.getOrientation().equals(Orientation.RIGHT)
|
||||
|
||||
&& ((textBlock.getMostPopularWordStyle().contains("bold") && Character.isDigit(textBlock.toString().charAt(0)))
|
||||
|| textBlock.toString().equals(textBlock.toString().toUpperCase(Locale.ROOT))
|
||||
|| textBlock.toString().startsWith("APPENDIX") || textBlock.toString().startsWith("TABLE"))) {
|
||||
textBlock.setClassification("H 1");
|
||||
document.setHeadlines(true);
|
||||
|
||||
}
|
||||
if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock, page.isRotated()) && (document.getFontSizeCounter()
|
||||
else
|
||||
if(matcher.find() && PositionUtils.getApproxLineCount(textBlock) < 2.9 && !matcher2.find()) {
|
||||
textBlock.setClassification("H 2");
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
|
||||
else if (PositionUtils.isOverBodyTextFrame(bodyTextFrame, textBlock) && (document.getFontSizeCounter()
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
textBlock.setClassification("Header");
|
||||
@ -64,54 +141,36 @@ public class ClassificationService {
|
||||
.getMostPopular() == null || textBlock.getHighestFontSize() <= document.getFontSizeCounter()
|
||||
.getMostPopular())) {
|
||||
textBlock.setClassification("Footer");
|
||||
} else if (page.getPageNumber() == 1 && (!PositionUtils.isTouchingUnderBodyTextFrame(bodyTextFrame, textBlock) && PositionUtils
|
||||
.getHeightDifferenceBetweenChunkWordAndDocumentWord(textBlock, document.getTextHeightCounter()
|
||||
.getMostPopular()) > 2.5 && textBlock.getHighestFontSize() > document.getFontSizeCounter()
|
||||
.getMostPopular() || page.getTextBlocks().size() == 1)) {
|
||||
if (!Pattern.matches("[0-9]+", textBlock.toString())) {
|
||||
textBlock.setClassification("Title");
|
||||
}
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && PositionUtils.getApproxLineCount(textBlock) < 4.9 && (textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") || !document.getFontStyleCounter().getCountPerValue().containsKey("bold") && textBlock.getMostPopularWordFontSize() > document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() + 1) && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
|
||||
for (int i = 1; i <= headlineFontSizes.size(); i++) {
|
||||
if (textBlock.getMostPopularWordFontSize() == headlineFontSizes.get(i - 1)) {
|
||||
textBlock.setClassification("H " + i);
|
||||
document.setHeadlines(true);
|
||||
}
|
||||
}
|
||||
} else if (!textBlock.getText().startsWith("Table ") && !textBlock.getText()
|
||||
.startsWith("Figure ") && PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
document.setHeadlines(true);
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
textBlock.setClassification("TextBlock Bold");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
.equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
.equals(document.getFontStyleCounter()
|
||||
.getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
.getMostPopular()) {
|
||||
// else if (textBlock.getMostPopularWordStyle().equals("bold") && !document.getFontStyleCounter()
|
||||
// .getMostPopular()
|
||||
// .equals("bold") && PositionUtils.getApproxLineCount(textBlock) < 2.9 && textBlock.getSequences().get(0).getTextPositions().get(0).getFontSizeInPt() >= textBlock.getMostPopularWordFontSize()) {
|
||||
// textBlock.setClassification("H " + (headlineFontSizes.size() + 1));
|
||||
// document.setHeadlines(true);
|
||||
// }
|
||||
// else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
// .getFontSizeCounter()
|
||||
// .getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
// .equals("bold") && !document.getFontStyleCounter().getMostPopular().equals("bold")) {
|
||||
// textBlock.setClassification("TextBlock Bold");
|
||||
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFont()
|
||||
// .equals(document.getFontCounter().getMostPopular()) && textBlock.getMostPopularWordStyle()
|
||||
// .equals(document.getFontStyleCounter()
|
||||
// .getMostPopular()) && textBlock.getMostPopularWordFontSize() == document.getFontSizeCounter()
|
||||
// .getMostPopular()) {
|
||||
// textBlock.setClassification("TextBlock");
|
||||
// } else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
// .getFontSizeCounter()
|
||||
// .getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
// .equals("italic") && !document.getFontStyleCounter()
|
||||
// .getMostPopular()
|
||||
// .equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
// textBlock.setClassification("TextBlock Italic");
|
||||
else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
textBlock.setClassification("TextBlock");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock) && textBlock.getMostPopularWordFontSize() == document
|
||||
.getFontSizeCounter()
|
||||
.getMostPopular() && textBlock.getMostPopularWordStyle()
|
||||
.equals("italic") && !document.getFontStyleCounter()
|
||||
.getMostPopular()
|
||||
.equals("italic") && PositionUtils.getApproxLineCount(textBlock) < 2.9) {
|
||||
textBlock.setClassification("TextBlock Italic");
|
||||
} else if (PositionUtils.isWithinBodyTextFrame(bodyTextFrame, textBlock)) {
|
||||
textBlock.setClassification("TextBlock Unknown");
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
textBlock.setClassification("Other");
|
||||
}
|
||||
}
|
||||
|
||||
@ -31,16 +31,16 @@ public class PositionUtils {
|
||||
}
|
||||
|
||||
|
||||
public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock, boolean rotated) {
|
||||
public boolean isOverBodyTextFrame(Rectangle btf, TextBlock textBlock) {
|
||||
|
||||
if (btf == null || textBlock == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (rotated && textBlock.getMinX() < btf.getX()) {
|
||||
if ((textBlock.getRotation() == 0 || textBlock.getRotation() == 180) && textBlock.getMinX() < btf.getX()) {
|
||||
// Its very strange, P{0,0} is on top left in this case, instead of lower left.
|
||||
return true;
|
||||
} else if (!rotated && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
|
||||
} else if ((textBlock.getRotation() == 90 || textBlock.getRotation() == 270) && textBlock.getMinY() > btf.getY() + btf.getHeight()) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@ -89,6 +89,9 @@ public class PositionUtils {
|
||||
|
||||
|
||||
public Float getApproxLineCount(TextBlock textBlock) {
|
||||
|
||||
// float height = textBlock.getRotation() == 0 || textBlock.getRotation() == 180 ? textBlock.getHeight() : textBlock.getWidth();
|
||||
|
||||
return textBlock.getHeight() / textBlock.getMostPopularWordHeight();
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,9 +9,12 @@ import java.nio.file.attribute.FileAttribute;
|
||||
import java.nio.file.attribute.PosixFilePermission;
|
||||
import java.nio.file.attribute.PosixFilePermissions;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.lang3.SystemUtils;
|
||||
@ -29,6 +32,7 @@ import com.iqser.red.service.redaction.v1.server.classification.service.Classifi
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.PDFLinesTextStripper;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextPositionSequence;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.model.PdfImage;
|
||||
import com.iqser.red.service.redaction.v1.server.redaction.utils.Patterns;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.CleanRulings;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.service.RulingCleaningService;
|
||||
@ -89,24 +93,41 @@ public class PdfSegmentationService {
|
||||
stripper.getText(pdDocument);
|
||||
|
||||
PDRectangle pdr = pdPage.getMediaBox();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight();
|
||||
|
||||
int rotation = pdPage.getRotation();
|
||||
boolean isLandscape = pdr.getWidth() > pdr.getHeight() && (rotation == 0 || rotation == 180) ||
|
||||
pdr.getHeight() > pdr.getWidth() && (rotation == 90 || rotation == 270);
|
||||
|
||||
|
||||
|
||||
System.out.println("PageNr: " + pageNumber + " rotation: "+rotation);
|
||||
|
||||
boolean isRotated = rotation != 0 && rotation != 360;
|
||||
|
||||
CleanRulings cleanRulings = rulingCleaningService.getCleanRulings(stripper.getRulings(), stripper.getMinCharWidth(), stripper
|
||||
.getMaxCharHeight());
|
||||
|
||||
// var sorted = stripper.getTextPositionSequences().stream().sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getXDirAdj())).sorted(Comparator.comparing(a -> a.getTextPositions().get(0).getYDirAdj())).sorted(Comparator.comparing(a -> a.getPage())).collect(Collectors.toList());
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Page page = blockificationService.blockify(stripper.getTextPositionSequences(), cleanRulings.getHorizontal(), cleanRulings
|
||||
.getVertical());
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinX()));
|
||||
|
||||
PDRectangle cropbox = pdPage.getCropBox();
|
||||
float cropboxArea = cropbox.getHeight() * cropbox.getWidth();
|
||||
page.setCropBoxArea(cropboxArea);
|
||||
|
||||
page.setRotation(rotation);
|
||||
page.setLandscape(isLandscape || isRotated);
|
||||
page.setLandscape(isLandscape);
|
||||
page.setPageNumber(pageNumber);
|
||||
|
||||
|
||||
@ -119,11 +140,34 @@ public class PdfSegmentationService {
|
||||
imageService.findOcr(page);
|
||||
}
|
||||
|
||||
|
||||
var pattern = Patterns.getCompiledPattern("^\\d[\\d.]{0,5}", true);
|
||||
page.getTextBlocks().sort(Comparator.comparing(a -> a.getMinY(), Comparator.reverseOrder()));
|
||||
|
||||
AbstractTextContainer prev = null;
|
||||
List<AbstractTextContainer> toRemove = new ArrayList<>();
|
||||
for (AbstractTextContainer current : page.getTextBlocks()) {
|
||||
|
||||
if(prev != null && prev instanceof TextBlock && current instanceof TextBlock && Math.abs(prev.getMinY() - current.getMinY()) <=1){
|
||||
Matcher matcher = pattern.matcher(prev.getText());
|
||||
if (matcher.matches()) {
|
||||
((TextBlock) current).getSequences().add(0, ((TextBlock) prev).getSequences().get(0));
|
||||
toRemove.add(prev);
|
||||
current.setMinX(prev.getMinX());
|
||||
}
|
||||
}
|
||||
prev = current;
|
||||
}
|
||||
|
||||
page.getTextBlocks().removeAll(toRemove);
|
||||
|
||||
|
||||
pages.add(page);
|
||||
}
|
||||
|
||||
document.setPages(pages);
|
||||
|
||||
|
||||
classificationService.classifyDocument(document);
|
||||
sectionsBuilderService.buildSections(document);
|
||||
sectionsBuilderService.addImagesToSections(document);
|
||||
|
||||
@ -68,7 +68,7 @@ public class SectionsBuilderService {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (prev != null && current.getClassification().startsWith("H ") && !prev.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
if (prev != null && current.getClassification().startsWith("H ") || !document.isHeadlines()) {
|
||||
Paragraph chunkBlock = buildTextBlock(chunkWords, lastHeadline);
|
||||
chunkBlock.setHeadline(lastHeadline);
|
||||
if (document.isHeadlines()) {
|
||||
|
||||
@ -1,9 +1,13 @@
|
||||
package com.iqser.red.service.redaction.v1.server.visualization.service;
|
||||
|
||||
import com.iqser.red.service.redaction.v1.model.Point;
|
||||
import com.iqser.red.service.redaction.v1.model.Rectangle;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Document;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Page;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.Paragraph;
|
||||
import com.iqser.red.service.redaction.v1.server.classification.model.TextBlock;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.RedTextPosition;
|
||||
import com.iqser.red.service.redaction.v1.server.parsing.model.TextDirection;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.AbstractTextContainer;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Cell;
|
||||
import com.iqser.red.service.redaction.v1.server.tableextraction.model.Table;
|
||||
@ -16,6 +20,8 @@ import org.apache.pdfbox.pdmodel.font.PDType1Font;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.awt.Color;
|
||||
import java.awt.geom.AffineTransform;
|
||||
import java.awt.geom.Point2D;
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
|
||||
@ -32,6 +38,7 @@ public class PdfVisualisationService {
|
||||
PDPage pdPage = document.getPage(page - 1);
|
||||
PDPageContentStream contentStream = new PDPageContentStream(document, pdPage, PDPageContentStream.AppendMode.APPEND, true);
|
||||
|
||||
int a = 0;
|
||||
for (Paragraph paragraph : classifiedDoc.getParagraphs()) {
|
||||
|
||||
for (int i = 0; i <= paragraph.getPageBlocks().size() - 1; i++) {
|
||||
@ -43,7 +50,8 @@ public class PdfVisualisationService {
|
||||
}
|
||||
if (textBlock instanceof TextBlock) {
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream);
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream, a);
|
||||
a++;
|
||||
} else if (textBlock instanceof Table) {
|
||||
textBlock.setClassification((i + 1) + "/" + paragraph.getPageBlocks().size());
|
||||
visualizeTable((Table) textBlock, contentStream);
|
||||
@ -59,7 +67,9 @@ public class PdfVisualisationService {
|
||||
|
||||
public void visualizeClassifications(Document classifiedDoc, PDDocument document) throws IOException {
|
||||
|
||||
|
||||
for (int page = 1; page <= document.getNumberOfPages(); page++) {
|
||||
int a = 0;
|
||||
|
||||
Page analyzedPage = classifiedDoc.getPages().get(page - 1);
|
||||
|
||||
@ -72,14 +82,20 @@ public class PdfVisualisationService {
|
||||
continue;
|
||||
}
|
||||
if (textBlock instanceof TextBlock) {
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream);
|
||||
visualizeTextBlock((TextBlock) textBlock, contentStream, a);
|
||||
a++;
|
||||
} else if (textBlock instanceof Table) {
|
||||
visualizeTable((Table) textBlock, contentStream);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
contentStream.setStrokingColor(Color.YELLOW);
|
||||
contentStream.addRect((float) analyzedPage.getBodyTextFrame().getX(), (float) analyzedPage.getBodyTextFrame().getY(), (float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight());
|
||||
|
||||
var rect = getRectangle((float) analyzedPage.getBodyTextFrame().getX(),(float) analyzedPage.getBodyTextFrame().getY(),(float) analyzedPage.getBodyTextFrame().getWidth(), (float) analyzedPage.getBodyTextFrame().getHeight(), pdPage.getRotation(), pdPage.getCropBox().getWidth(), pdPage.getCropBox().getHeight());
|
||||
|
||||
contentStream.addRect(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
|
||||
|
||||
contentStream.stroke();
|
||||
|
||||
contentStream.close();
|
||||
@ -87,28 +103,63 @@ public class PdfVisualisationService {
|
||||
}
|
||||
|
||||
|
||||
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream) throws IOException {
|
||||
private void visualizeTextBlock(TextBlock textBlock, PDPageContentStream contentStream, int number) throws IOException {
|
||||
|
||||
contentStream.setStrokingColor(Color.RED);
|
||||
|
||||
contentStream.addRect(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight());
|
||||
var rect = getRectangle(textBlock.getMinX(), textBlock.getMinY(), textBlock.getWidth(), textBlock.getHeight(), textBlock.getRotation(), textBlock.getSequences().get(0).getPageWidth(), textBlock.getSequences().get(0).getPageHeight());
|
||||
|
||||
contentStream.addRect(rect.getTopLeft().getX(), rect.getTopLeft().getY(), rect.getWidth(), rect.getHeight());
|
||||
contentStream.stroke();
|
||||
|
||||
if (textBlock.getClassification() != null) {
|
||||
contentStream.beginText();
|
||||
|
||||
contentStream.setNonStrokingColor(Color.BLUE);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 12f);
|
||||
contentStream.setFont(PDType1Font.TIMES_ROMAN, 6f);
|
||||
|
||||
contentStream.newLineAtOffset(textBlock.getMinX(), textBlock.getMaxY());
|
||||
contentStream.newLineAtOffset(rect.getTopLeft().getX(), rect.getTopLeft().getY());
|
||||
|
||||
contentStream.showText(textBlock.getClassification() + textBlock.getOrientation());
|
||||
contentStream.showText(textBlock.getClassification() /*" textBlock.getOrientation() + "--> "*/ + number);
|
||||
|
||||
contentStream.endText();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Rectangle getRectangle(float x, float y, float width, float height, int rotation, float pageWidth, float pageHeight) {
|
||||
|
||||
|
||||
Point2D bottomLeft = new Point2D.Double(x, pageHeight - y);
|
||||
Point2D topRight = new Point2D.Double(x + width,pageHeight - y - height);
|
||||
|
||||
AffineTransform transform = new AffineTransform();
|
||||
if (rotation == 0 || rotation == 180) {
|
||||
transform.rotate(Math.toRadians(rotation), pageWidth / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageHeight);
|
||||
transform.scale(1., -1.);
|
||||
} else if (rotation == 90) {
|
||||
transform.rotate(Math.toRadians(rotation), pageWidth / 2f, pageWidth / 2f);
|
||||
transform.translate(0f, pageWidth);
|
||||
transform.scale(1., -1.);
|
||||
} else {
|
||||
transform.rotate(Math.toRadians(rotation), pageHeight / 2f, pageHeight / 2f);
|
||||
transform.translate(0f, pageWidth);
|
||||
transform.scale(1., -1.);
|
||||
}
|
||||
|
||||
bottomLeft = transform.transform(bottomLeft, null);
|
||||
topRight = transform.transform(topRight, null);
|
||||
|
||||
return new Rectangle( //
|
||||
new Point((float) bottomLeft.getX(), (float) bottomLeft.getY()), (float) (topRight.getX() - bottomLeft.getX()), (float) (topRight.getY() - bottomLeft.getY()), 0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private void visualizeTable(Table table, PDPageContentStream contentStream) throws IOException {
|
||||
for (List<Cell> row : table.getRows()) {
|
||||
for (Cell cell : row) {
|
||||
|
||||
@ -334,7 +334,9 @@ public class RedactionIntegrationTest {
|
||||
@Test
|
||||
public void titleExtraction() throws IOException {
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/32 - Emamectin Benzoate Technical - Acute Oral Toxicity - Mouse.pdf");
|
||||
System.out.println("Detailed quantitative and qualitative information on the composition of the plant ".length());
|
||||
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/Metolachlor/S-Metolachlor_RAR_02_Volume_2_2018-09-06.pdf");
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
analyzeService.analyzeDocumentStructure(new StructureAnalyzeRequest(request.getDossierId(), request.getFileId()));
|
||||
@ -1094,7 +1096,7 @@ public class RedactionIntegrationTest {
|
||||
public void classificationTest() throws IOException {
|
||||
|
||||
System.out.println("classificationTest");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/01 - CGA100251 - Acute Oral Toxicity (Up and Down Procedure) - Rat (1).pdf");
|
||||
ClassPathResource pdfFileResource = new ClassPathResource("files/RSS/02 - A22833B - Acute Oral (Up and Down) - Final Report.pdf");
|
||||
|
||||
AnalyzeRequest request = prepareStorage(pdfFileResource.getInputStream());
|
||||
|
||||
|
||||
@ -15,379 +15,9 @@ global Section section
|
||||
// end
|
||||
|
||||
|
||||
rule "0: Add CBI_author from ai"
|
||||
rule "1: Find headlines "
|
||||
when
|
||||
Section(aiMatchesType("CBI_author"))
|
||||
Section(text.length() > 1)
|
||||
then
|
||||
section.addAiEntities("CBI_author", "CBI_author");
|
||||
end
|
||||
|
||||
rule "0: Combine ai types CBI_author from ai"
|
||||
when
|
||||
Section(aiMatchesType("ORG"))
|
||||
then
|
||||
section.combineAiTypes("ORG", "STREET,POSTAL,COUNTRY,CARDINAL,CITY,STATE", 20, "CBI_address", 3, false);
|
||||
end
|
||||
|
||||
rule "0: Expand CBI Authors with firstname initials"
|
||||
when
|
||||
Section(matchesType("CBI_author"))
|
||||
then
|
||||
section.expandByRegEx("CBI_author", "(,? [A-Z]\\.?( ?[A-Z]\\.?)?( ?[A-Z]\\.?)?\\b\\.?)", false, 1, "[^\\s]+");
|
||||
end
|
||||
|
||||
rule "0: Expand CBI_author and PII matches with salutation prefix"
|
||||
when
|
||||
Section((matchesType("CBI_author") || matchesType("PII")) && (
|
||||
searchText.contains("Mr")
|
||||
|| searchText.contains("Mrs")
|
||||
|| searchText.contains("Ms")
|
||||
|| searchText.contains("Miss")
|
||||
|| searchText.contains("Sir")
|
||||
|| searchText.contains("Madam")
|
||||
|| searchText.contains("Madame")
|
||||
|| searchText.contains("Mme")
|
||||
))
|
||||
then
|
||||
section.expandByPrefixRegEx("CBI_author", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
||||
section.expandByPrefixRegEx("PII", "\\b(Mrs?|Ms|Miss|Sir|Madame?|Mme)\\s?\\.?\\s*", false, 0);
|
||||
end
|
||||
|
||||
|
||||
rule "1: Redacted because Section contains Vertebrate"
|
||||
when
|
||||
Section(matchesType("vertebrate"))
|
||||
then
|
||||
section.redact("CBI_author", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 1, "Vertebrate found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "2: Not Redacted because Section contains no Vertebrate"
|
||||
when
|
||||
Section(!matchesType("vertebrate"))
|
||||
then
|
||||
section.redactNot("CBI_author", 2, "No Vertebrate found");
|
||||
section.redactNot("CBI_address", 2, "No Vertebrate found");
|
||||
end
|
||||
|
||||
|
||||
rule "3: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"))
|
||||
then
|
||||
section.redactNot("CBI_author", 3, "Vertebrate and No Redaction Indicator found");
|
||||
section.redactNot("CBI_address", 3, "Vertebrate and No Redaction Indicator found");
|
||||
end
|
||||
|
||||
|
||||
rule "4: Redact Names and Addresses if no_redaction_indicator and redaction_indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("no_redaction_indicator"), matchesType("redaction_indicator"))
|
||||
then
|
||||
section.redact("CBI_author", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 4, "Vertebrate and Redaction Indicator found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "5: Do not redact Names and Addresses if no redaction Indicator is contained"
|
||||
when
|
||||
Section(matchesType("vertebrate"), matchesType("published_information"))
|
||||
then
|
||||
section.redactNotAndReference("CBI_author","published_information", 5, "Vertebrate and Published Information found");
|
||||
section.redactNotAndReference("CBI_address","published_information", 5, "Vertebrate and Published Information found");
|
||||
end
|
||||
|
||||
|
||||
rule "6: Not redacted because Vertebrate Study = N"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "N") || rowEquals("Vertebrate study Y/N", "No"))
|
||||
then
|
||||
section.redactNotCell("Author(s)", 6, "CBI_author", true, "Not redacted because row is not a vertebrate study");
|
||||
section.redactNot("CBI_author", 6, "Not redacted because row is not a vertebrate study");
|
||||
section.redactNot("CBI_address", 6, "Not redacted because row is not a vertebrate study");
|
||||
section.highlightCell("Vertebrate study Y/N", 6, "hint_only");
|
||||
end
|
||||
|
||||
|
||||
rule "7: Redact if must redact entry is found"
|
||||
when
|
||||
Section(matchesType("must_redact"))
|
||||
then
|
||||
section.redact("CBI_author", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 7, "must_redact entry was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "8: Redact Authors and Addresses in Reference Table if it is a Vertebrate study"
|
||||
when
|
||||
Section(rowEquals("Vertebrate study Y/N", "Y") || rowEquals("Vertebrate study Y/N", "Yes"))
|
||||
then
|
||||
section.redactCell("Author(s)", 8, "CBI_author", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 8, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.highlightCell("Vertebrate study Y/N", 8, "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "9: Redact sponsor company"
|
||||
when
|
||||
Section(searchText.toLowerCase().contains("batches produced at"))
|
||||
then
|
||||
section.redactIfPrecededBy("batches produced at", "CBI_sponsor", 9, "Redacted because it represents a sponsor company", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("batches produced at", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "10: Redact determination of residues"
|
||||
when
|
||||
Section((
|
||||
searchText.toLowerCase.contains("determination of residues") ||
|
||||
searchText.toLowerCase.contains("determination of total residues")
|
||||
) && (
|
||||
searchText.toLowerCase.contains("livestock") ||
|
||||
searchText.toLowerCase.contains("live stock") ||
|
||||
searchText.toLowerCase.contains("tissue") ||
|
||||
searchText.toLowerCase.contains("tissues") ||
|
||||
searchText.toLowerCase.contains("liver") ||
|
||||
searchText.toLowerCase.contains("muscle") ||
|
||||
searchText.toLowerCase.contains("bovine") ||
|
||||
searchText.toLowerCase.contains("ruminant") ||
|
||||
searchText.toLowerCase.contains("ruminants")
|
||||
))
|
||||
then
|
||||
section.redact("CBI_author", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 10, "Determination of residues was found.", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("determination of residues", "must_redact");
|
||||
section.addHintAnnotation("livestock", "must_redact");
|
||||
section.addHintAnnotation("live stock", "must_redact");
|
||||
section.addHintAnnotation("tissue", "must_redact");
|
||||
section.addHintAnnotation("tissues", "must_redact");
|
||||
section.addHintAnnotation("liver", "must_redact");
|
||||
section.addHintAnnotation("muscle", "must_redact");
|
||||
section.addHintAnnotation("bovine", "must_redact");
|
||||
section.addHintAnnotation("ruminant", "must_redact");
|
||||
section.addHintAnnotation("ruminants", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "11: Redact if CTL/* or BL/* was found"
|
||||
when
|
||||
Section(searchText.contains("CTL/") || searchText.contains("BL/"))
|
||||
then
|
||||
section.redact("CBI_author", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redact("CBI_address", 11, "Laboraty for vertebrate studies found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.addHintAnnotation("CTL", "must_redact");
|
||||
section.addHintAnnotation("BL", "must_redact");
|
||||
end
|
||||
|
||||
|
||||
rule "12: Redact and add recommendation for et al. author"
|
||||
when
|
||||
Section(searchText.contains("et al"))
|
||||
then
|
||||
section.redactAndRecommendByRegEx("\\b([A-ZÄÖÜ][^\\s\\.,]+( [A-ZÄÖÜ]{1,2}\\.?)?( ?[A-ZÄÖÜ]\\.?)?) et al\\.?", false, 1, "CBI_author", 12, "Author found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "13: Add recommendation for Addresses in Test Organism sections"
|
||||
when
|
||||
Section(searchText.contains("Species:") && searchText.contains("Source:"))
|
||||
then
|
||||
section.recommendLineAfter("Source:", "CBI_address");
|
||||
end
|
||||
|
||||
|
||||
rule "14: Add recommendation for Addresses in Test Animals sections"
|
||||
when
|
||||
Section(searchText.contains("Species") && searchText.contains("Source"))
|
||||
then
|
||||
section.recommendLineAfter("Source", "CBI_address");
|
||||
end
|
||||
|
||||
// --------------------------------------- PII rules -------------------------------------------------------------------
|
||||
|
||||
|
||||
rule "14: Redacted PII Personal Identification Information"
|
||||
when
|
||||
Section(matchesType("PII"))
|
||||
then
|
||||
section.redact("PII", 14, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "15: Redact Emails by RegEx"
|
||||
when
|
||||
Section(searchText.contains("@"))
|
||||
then
|
||||
section.redactByRegEx("\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,4}\\b", true, 0, "PII", 15, "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "16: Redact contact information"
|
||||
when
|
||||
Section(text.contains("Contact point:")
|
||||
|| text.contains("Phone:")
|
||||
|| text.contains("Fax:")
|
||||
|| text.contains("Tel.:")
|
||||
|| text.contains("Tel:")
|
||||
|| text.contains("E-mail:")
|
||||
|| text.contains("Email:")
|
||||
|| text.contains("e-mail:")
|
||||
|| text.contains("E-mail address:")
|
||||
|| text.contains("Alternative contact:")
|
||||
|| text.contains("Telephone number:")
|
||||
|| text.contains("Telephone No:")
|
||||
|| text.contains("Fax number:")
|
||||
|| text.contains("Telephone:")
|
||||
|| text.contains("European contact:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 16, true, "Contact information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "17: Redact contact information if applicant is found"
|
||||
when
|
||||
Section(headlineContainsWord("applicant") || text.contains("Applicant") || headlineContainsWord("Primary contact") || headlineContainsWord("Alternative contact") || text.contains("Telephone number:"))
|
||||
then
|
||||
section.redactLineAfter("Contact point:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Email:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("e-mail:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail address:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Alternative contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone No:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("Contact:", "Tel.:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("European contact:", "PII", 17, true, "Applicant information was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "18: Redact contact information if Producer is found"
|
||||
when
|
||||
Section(text.toLowerCase().contains("producer of the plant protection") || text.toLowerCase().contains("producer of the active substance") || text.contains("Manufacturer of the active substance") || text.contains("Manufacturer:") || text.contains("Producer or producers of the active substance"))
|
||||
then
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Phone:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("E-mail:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Contact:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Fax number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Telephone number:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactLineAfter("Tel:", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
section.redactBetween("No:", "Fax", "PII", 18, true, "Producer was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "19: Redact AUTHOR(S)"
|
||||
when
|
||||
Section(searchText.contains("AUTHOR(S):") && fileAttributeByPlaceholderEquals("{fileattributes.vertebrateStudy}", "true"))
|
||||
then
|
||||
section.redactLinesBetween("AUTHOR(S):", "COMPLETION DATE:", "PII", 19, true, "AUTHOR(S) was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "20: Redact PERFORMING LABORATORY"
|
||||
when
|
||||
Section(searchText.contains("PERFORMING LABORATORY:"))
|
||||
then
|
||||
section.redactBetween("PERFORMING LABORATORY:", "LABORATORY PROJECT ID:", "PII", 20, true, "PERFORMING LABORATORY was found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "21: Redact On behalf of Sequani Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Sequani Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Sequani Ltd.: Name Title", "On behalf of", "PII", 21, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
|
||||
rule "22: Redact On behalf of Syngenta Ltd.:"
|
||||
when
|
||||
Section(searchText.contains("On behalf of Syngenta Ltd.: Name Title"))
|
||||
then
|
||||
section.redactBetween("On behalf of Syngenta Ltd.: Name Title", "Study dates", "PII", 22, false , "PII (Personal Identification Information) found", "Reg (EC) No 1107/2009 Art. 63 (2e)");
|
||||
end
|
||||
|
||||
// --------------------------------------- other rules -------------------------------------------------------------------
|
||||
|
||||
rule "25: Redact Purity"
|
||||
when
|
||||
Section(searchText.contains("purity"))
|
||||
then
|
||||
section.redactByRegEx("purity ?:? (([\\d\\.]+)( .{0,4}\\.)? ?%)", true, 1, "purity", 17, "Purity found", "Reg (EC) No 1107/2009 Art. 63 (2a)");
|
||||
end
|
||||
|
||||
|
||||
rule "26: Redact signatures"
|
||||
when
|
||||
Section(matchesImageType("signature"))
|
||||
then
|
||||
section.redactImage("signature", 26, "Signature found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "27: Redact formula"
|
||||
when
|
||||
Section(matchesImageType("formula"))
|
||||
then
|
||||
section.redactImage("formula", 27, "Formula found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "28: Redact Logos"
|
||||
when
|
||||
Section(matchesImageType("logo"))
|
||||
then
|
||||
section.redactImage("logo", 28, "Logo found", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
end
|
||||
|
||||
|
||||
rule "29: Redact Dossier Redactions"
|
||||
when
|
||||
Section(matchesType("dossier_redactions"))
|
||||
then
|
||||
section.redact("dossier_redactions", 29, "Dossier Redaction found", "Article 39(1)(2) of Regulation (EC) No 178/2002");
|
||||
end
|
||||
|
||||
rule "30: Ignore dossier_redactions if confidential"
|
||||
when
|
||||
Section(!fileAttributeByLabelEqualsIgnoreCase("Confidentiality","confidential") && matchesType("dossier_redactions"));
|
||||
then
|
||||
section.ignore("dossier_redactions");
|
||||
end
|
||||
|
||||
// ex. "New Rules for PAD" - "Annex A" - page 21, page 35 (table without header), page 38 (in-text)
|
||||
// https://www.regexplanet.com/share/index.html?share=yyyypb71xkr
|
||||
rule "101: Redact CAS numbers"
|
||||
when
|
||||
Section(hasTableHeader("Sample #"))
|
||||
then
|
||||
section.redactCell("Sample #", 8, "PII", true, "Redacted because row is a vertebrate study", "Reg (EC) No 1107/2009 Art. 63 (2g)");
|
||||
section.redactHeadline("PII", 1, "Headline found", "n-a.");
|
||||
end
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user