memory optimisations

This commit is contained in:
Timo Bejan 2024-03-14 18:50:07 +02:00
parent e926083881
commit 1b4ab8dc88
6 changed files with 283 additions and 199 deletions

View File

@ -1,7 +1,5 @@
package com.iqser.red.pdftronlogic.commons; package com.iqser.red.pdftronlogic.commons;
import static com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService.TOLERANCE; import static com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService.TOLERANCE;
import java.awt.geom.Area; import java.awt.geom.Area;
@ -35,6 +33,13 @@ public class ClippingPathStack {
} }
@SneakyThrows
public void intersectClippingPath(Rectangle2D path) {
getCurrentClippingPath().intersect(new Area(path));
}
public boolean almostIntersects(double x, double y, double width, double height) { public boolean almostIntersects(double x, double y, double width, double height) {
// To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle // To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle
// Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0. // Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0.
@ -56,15 +61,16 @@ public class ClippingPathStack {
public void enterNewGState() { public void enterNewGState() {
Area current = stack.peek(); Area current = stack.peek();
Area cloned = new Area(); Area cloned = (Area) current.clone();
cloned.add(current);
stack.push(cloned); stack.push(cloned);
} }
public void leaveGState() { public void leaveGState() {
stack.pop(); // somehow this greatly helps memory management
var popped = stack.pop();
popped.reset();
} }
} }

View File

@ -4,6 +4,7 @@ import com.pdftron.common.PDFNetException;
import com.pdftron.pdf.Element; import com.pdftron.pdf.Element;
public class ElementFeatureFactory { public class ElementFeatureFactory {
public static ElementFeatures extractFeatures(Element element) throws PDFNetException { public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
return switch (element.getType()) { return switch (element.getType()) {
@ -16,7 +17,9 @@ public class ElementFeatureFactory {
}; };
} }
public static ElementFeatures extractFeaturesWithHash(Element element, String hashObject) throws PDFNetException { public static ElementFeatures extractFeaturesWithHash(Element element, String hashObject) throws PDFNetException {
return buildImage(element) return buildImage(element)
.hashOfImage(hashObject) .hashOfImage(hashObject)
.build(); .build();
@ -25,20 +28,23 @@ public class ElementFeatureFactory {
private static ElementFeatures.Form buildForm(Element element) throws PDFNetException { private static ElementFeatures.Form buildForm(Element element) throws PDFNetException {
try (var bbox = element.getBBox();) {
return ElementFeatures.Form.builder() return ElementFeatures.Form.builder()
.elementType(element.getType()) .elementType(element.getType())
.boundingBox(Converter.toRectangle2D(element.getBBox())) .boundingBox(Converter.toRectangle2D(bbox))
.xObjectType(element.getXObject().getType()) .xObjectType(element.getXObject().getType())
.dictOrArrayOrStreamLength(element.getXObject().getType() == 7 ? element.getXObject().getDecodedStream().size() : 0) .dictOrArrayOrStreamLength(element.getXObject().getType() == 7 ? element.getXObject().getDecodedStream().size() : 0)
.build(); .build();
} }
}
private static ElementFeatures.Image.ImageBuilder<?, ?> buildImage(Element element) throws PDFNetException { private static ElementFeatures.Image.ImageBuilder<?, ?> buildImage(Element element) throws PDFNetException {
try (var bbox = element.getBBox();) {
return ElementFeatures.Image.builder() return ElementFeatures.Image.builder()
.elementType(element.getType()) .elementType(element.getType())
.boundingBox(Converter.toRectangle2D(element.getBBox())) .boundingBox(Converter.toRectangle2D(bbox))
.dataSize(element.getImageDataSize()) .dataSize(element.getImageDataSize())
.height(element.getImageHeight()) .height(element.getImageHeight())
.width(element.getImageWidth()) .width(element.getImageWidth())
@ -46,35 +52,41 @@ public class ElementFeatureFactory {
.componentNum(element.getComponentNum()) .componentNum(element.getComponentNum())
.bitsPerComponent(element.getBitsPerComponent()); .bitsPerComponent(element.getBitsPerComponent());
} }
}
private static ElementFeatures.Text buildText(Element element) throws PDFNetException { private static ElementFeatures.Text buildText(Element element) throws PDFNetException {
try (var bbox = element.getBBox();) {
return ElementFeatures.Text.builder() return ElementFeatures.Text.builder()
.elementType(element.getType()) .elementType(element.getType())
.boundingBox(Converter.toRectangle2D(element.getBBox())) .boundingBox(Converter.toRectangle2D(bbox))
.text(element.getTextString()) .text(element.getTextString())
.font(element.getGState().getFont().getType()) .font(element.getGState().getFont().getType())
.fontsize(element.getGState().getFontSize()) .fontsize(element.getGState().getFontSize())
.build(); .build();
} }
}
private static ElementFeatures.Path buildPath(Element element) throws PDFNetException { private static ElementFeatures.Path buildPath(Element element) throws PDFNetException {
try (var bbox = element.getBBox(); var ctm = element.getCTM();
var fillColor = element.getGState().getFillColor();
var strokeColor = element.getGState().getStrokeColor()) {
return ElementFeatures.Path.builder() return ElementFeatures.Path.builder()
.elementType(element.getType()) .elementType(element.getType())
.boundingBox(Converter.toRectangle2D(element.getBBox())) .boundingBox(Converter.toRectangle2D(bbox))
.isClippingPath(element.isClippingPath()) .isClippingPath(element.isClippingPath())
.isClipWindingFill(element.isClipWindingFill()) .isClipWindingFill(element.isClipWindingFill())
.isStroked(element.isStroked()) .isStroked(element.isStroked())
.isFilled(element.isFilled()) .isFilled(element.isFilled())
.isWindingFill(element.isWindingFill()) .isWindingFill(element.isWindingFill())
.fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), element.getGState().getFillColor())) .fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), fillColor))
.strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), element.getGState().getStrokeColor())) .strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), strokeColor))
.linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), element.getCTM())) .linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), ctm))
.build(); .build();
} }
}
} }

View File

@ -31,9 +31,11 @@ public class ElementFeatures {
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
try (var bbox = element.getBBox()) {
return element.getType() == elementType && // return element.getType() == elementType && //
element.getBBox() != null && // bbox != null && //
rectsAlmostMatch(element.getBBox()); rectsAlmostMatch(bbox);
}
} }
@ -199,7 +201,10 @@ public class ElementFeatures {
if (elementFeatures.getClass() != this.getClass()) { if (elementFeatures.getClass() != this.getClass()) {
return false; return false;
} }
return super.almostMatches(elementFeatures) && this.dataSize == ((Image) elementFeatures).getDataSize() && this.height == ((Image) elementFeatures).getHeight() && this.width == ((Image) elementFeatures).getWidth() && this.renderingIntent == ((Image) elementFeatures).getRenderingIntent() && this.componentNum == ((Image) elementFeatures).getComponentNum() && this.bitsPerComponent == ((Image) elementFeatures).getBitsPerComponent() && calculateHammingDistance( return super.almostMatches(elementFeatures) && this.dataSize == ((Image) elementFeatures).getDataSize() && this.height == ((Image) elementFeatures).getHeight() &&
this.width == ((Image) elementFeatures).getWidth() && this.renderingIntent == ((Image) elementFeatures).getRenderingIntent() &&
this.componentNum == ((Image) elementFeatures).getComponentNum() && this.bitsPerComponent == ((Image) elementFeatures).getBitsPerComponent() &&
calculateHammingDistance(
((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD; ((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD;
} }
@ -241,11 +246,13 @@ public class ElementFeatures {
@Override @Override
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
try (var bbox = element.getBBox()) {
return element.getType() == getElementType() && // return element.getType() == getElementType() && //
element.getBBox() != null && // bbox != null && //
(super.rectsAlmostMatch(element.getBBox()) || almostRotateMatches(element.getBBox().getRectangle())) && xObjectType == element.getXObject() (super.rectsAlmostMatch(bbox) || almostRotateMatches(bbox.getRectangle())) && xObjectType == element.getXObject()
.getType() && dictOrArrayOrStreamLength == element.getXObject().getDecodedStream().size(); .getType() && dictOrArrayOrStreamLength == element.getXObject().getDecodedStream().size();
} }
}
public boolean almostMatches(ElementFeatures elementFeatures) { public boolean almostMatches(ElementFeatures elementFeatures) {
@ -253,9 +260,11 @@ public class ElementFeatures {
if (elementFeatures.getClass() != this.getClass()) { if (elementFeatures.getClass() != this.getClass()) {
return false; return false;
} }
return elementFeatures.getElementType() == getElementType() && elementFeatures.getBoundingBox() != null && (super.rectsAlmostMatch(elementFeatures.getBoundingBox()) || almostRotateMatches( return elementFeatures.getElementType() == getElementType() && elementFeatures.getBoundingBox() != null &&
(super.rectsAlmostMatch(elementFeatures.getBoundingBox()) || almostRotateMatches(
elementFeatures.getBoundingBox() elementFeatures.getBoundingBox()
.getBounds2D())) && xObjectType == ((Form) elementFeatures).getXObjectType() && dictOrArrayOrStreamLength == ((Form) elementFeatures).getDictOrArrayOrStreamLength(); .getBounds2D())) && xObjectType == ((Form) elementFeatures).getXObjectType() &&
dictOrArrayOrStreamLength == ((Form) elementFeatures).getDictOrArrayOrStreamLength();
} }

View File

@ -63,15 +63,12 @@ public class InvisibleElementRemovalService {
PDFDoc pdfDoc = new PDFDoc(pdfFile); PDFDoc pdfDoc = new PDFDoc(pdfFile);
try (pdfDoc) {
execute(pdfDoc, delta, removePaths, markedContentToIgnore); execute(pdfDoc, delta, removePaths, markedContentToIgnore);
try {
pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null); pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
} catch (Exception e) { } catch (Exception e) {
log.error("File could not be saved after invisible element removal"); log.error("File could not be saved after invisible element removal");
throw new RuntimeException(e); throw new RuntimeException(e);
} finally {
pdfDoc.close();
} }
} }
@ -151,7 +148,7 @@ public class InvisibleElementRemovalService {
private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) { private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) {
log.info("Start removing invisible Elements"); log.info("Start removing invisible Elements");
try(ElementWriter writer = new ElementWriter(); try (ElementWriter writer = new ElementWriter();
ElementReader reader = new ElementReader()) { ElementReader reader = new ElementReader()) {
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
@ -236,7 +233,7 @@ public class InvisibleElementRemovalService {
private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException { private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
try(Rect rect = imageElement.getBBox()) { try (Rect rect = imageElement.getBBox()) {
if (rect == null) { if (rect == null) {
return; return;
@ -257,7 +254,7 @@ public class InvisibleElementRemovalService {
private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException { private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
try(Rect textBBox = textElement.getBBox()) { try (Rect textBBox = textElement.getBBox()) {
if (textBBox == null) { if (textBBox == null) {
writer.writeElement(textElement); writer.writeElement(textElement);
@ -289,13 +286,17 @@ public class InvisibleElementRemovalService {
if (!inClippingPath) { if (!inClippingPath) {
gState.setFillColorSpace(ColorSpace.createDeviceRGB()); gState.setFillColorSpace(ColorSpace.createDeviceRGB());
// red for elements removed by clipping path // red for elements removed by clipping path
gState.setFillColor(new ColorPt(1, 0, 0)); try (var color = new ColorPt(1, 0, 0)) {
gState.setFillColor(color);
}
writer.writeElement(textElement); writer.writeElement(textElement);
} }
if (!isTextVisible) { if (!isTextVisible) {
gState.setFillColorSpace(ColorSpace.createDeviceRGB()); gState.setFillColorSpace(ColorSpace.createDeviceRGB());
// blue for elements removed due to transparency or not rendered or same color as background // blue for elements removed due to transparency or not rendered or same color as background
gState.setFillColor(new ColorPt(0, 0, 1)); try (var color = new ColorPt(0, 0, 1)) {
gState.setFillColor(color);
}
gState.setTextRenderMode(GState.e_fill_text); gState.setTextRenderMode(GState.e_fill_text);
gState.setFillOpacity(1); gState.setFillOpacity(1);
writer.writeElement(textElement); writer.writeElement(textElement);
@ -314,9 +315,10 @@ public class InvisibleElementRemovalService {
context.visitedXObjIds().add(formObj.getObjNum()); context.visitedXObjIds().add(formObj.getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
context.clippingPathStack().enterNewGState(); context.clippingPathStack().enterNewGState();
context.clippingPathStack().intersectClippingPath(new GeneralPath(Converter.toRectangle2D(formElement.getBBox()))); try (var formElementBBOX = formElement.getBBox()) {
context.clippingPathStack().intersectClippingPath(Converter.toRectangle2D(formElementBBOX));
context.reader().formBegin(); context.reader().formBegin();
formWriter.begin(formObj); formWriter.begin(formObj);
@ -330,18 +332,21 @@ public class InvisibleElementRemovalService {
} }
} }
} }
}
private void processPath(Element pathElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException { private void processPath(Element pathElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
PathData pathData = pathElement.getPathData(); PathData pathData = pathElement.getPathData();
if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0 || pathElement.getBBox() == null) { try (var bbox = pathElement.getBBox()) {
if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0 || bbox == null) {
writer.writeElement(pathElement); writer.writeElement(pathElement);
return; return;
} }
GeneralPath linePath = Converter.convertToGeneralPathAndTransformToInitialUserSpace(pathData, pathElement.getCTM()); try (var ctm = pathElement.getCTM()) {
GeneralPath linePath = Converter.convertToGeneralPathAndTransformToInitialUserSpace(pathData, ctm);
var rect = linePath.getBounds2D(); var rect = linePath.getBounds2D();
@ -377,14 +382,18 @@ public class InvisibleElementRemovalService {
} }
if (context.delta() && !inClippingPath && context.removePaths()) { if (context.delta() && !inClippingPath && context.removePaths()) {
try (var color = new ColorPt(1, 0, 0)) {
pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB()); pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setFillColor(new ColorPt(1, 0, 0)); pathElement.getGState().setFillColor(color);
pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB()); pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setStrokeColor(new ColorPt(1, 0, 0)); pathElement.getGState().setStrokeColor(color);
writer.writeElement(pathElement); writer.writeElement(pathElement);
} }
} }
} }
}
}
}
private void calculateOverlapsForLinePath(InvisibleElementRemovalContext context, GeneralPath linePath) { private void calculateOverlapsForLinePath(InvisibleElementRemovalContext context, GeneralPath linePath) {
@ -480,7 +489,7 @@ public class InvisibleElementRemovalService {
context.visitedXObjIds().add(formObj.getObjNum()); context.visitedXObjIds().add(formObj.getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
context.reader().formBegin(); context.reader().formBegin();
formWriter.begin(formObj); formWriter.begin(formObj);
@ -516,7 +525,9 @@ public class InvisibleElementRemovalService {
private boolean fillIsVisible(GState gState, Rect textBBox, InvisibleElementRemovalContext context) throws PDFNetException { private boolean fillIsVisible(GState gState, Rect textBBox, InvisibleElementRemovalContext context) throws PDFNetException {
return gState.getFillOpacity() != 0 && differentColorThanBackgroundColor(Converter.convertColor(gState.getFillColorSpace(), gState.getFillColor()), textBBox, context); try (var color = gState.getFillColor()) {
return gState.getFillOpacity() != 0 && differentColorThanBackgroundColor(Converter.convertColor(gState.getFillColorSpace(), color), textBBox, context);
}
} }
@ -541,13 +552,16 @@ public class InvisibleElementRemovalService {
private static List<ElementFeatures.Path> findVisiblePathElementsThatIntersect(Rect textBBox, InvisibleElementRemovalContext context) { private static List<ElementFeatures.Path> findVisiblePathElementsThatIntersect(Rect textBBox, InvisibleElementRemovalContext context) {
return context.visibleElements() var result = new ArrayList<ElementFeatures.Path>();
.stream() for (var element : context.visibleElements()) {
.filter(elementFeatures -> elementFeatures.getElementType() == Element.e_path) if (element.getElementType() == Element.e_path
.map(elementFeatures -> (ElementFeatures.Path) elementFeatures) && !((ElementFeatures.Path) element).getFillColor().equals(Color.WHITE)
.filter(elementFeatures -> !elementFeatures.getFillColor().equals(Color.WHITE)) && ((ElementFeatures.Path) element).isBackground(textBBox)) {
.filter(element -> element.isBackground(textBBox)) result.add((ElementFeatures.Path) element);
.toList(); }
}
return result;
} }
@ -581,7 +595,7 @@ public class InvisibleElementRemovalService {
@SneakyThrows @SneakyThrows
private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) { private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) {
try(ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d, try (ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d,
Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d, Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d,
Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d); Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d);
ElementBuilder eb = new ElementBuilder()) { ElementBuilder eb = new ElementBuilder()) {

View File

@ -93,7 +93,7 @@ public class WatermarkRemovalService {
Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>(); Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>();
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
try(ElementReader reader = new ElementReader()) { try (ElementReader reader = new ElementReader()) {
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) { for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
@ -123,7 +123,8 @@ public class WatermarkRemovalService {
double minAreaCoveringPage, double minAreaCoveringPage,
Page page) throws PDFNetException { Page page) throws PDFNetException {
if (element.getBBox() == null) { try (var bbox = element.getBBox()) {
if (bbox == null) {
return; return;
} }
@ -132,6 +133,7 @@ public class WatermarkRemovalService {
case Element.e_image, Element.e_inline_image -> processImages(element, elementFeaturesLinkedList, page, minAreaCoveringPage); case Element.e_image, Element.e_inline_image -> processImages(element, elementFeaturesLinkedList, page, minAreaCoveringPage);
case Element.e_text -> processText(element, elementFeaturesLinkedList, page); case Element.e_text -> processText(element, elementFeaturesLinkedList, page);
} }
}
} }
@ -147,12 +149,14 @@ public class WatermarkRemovalService {
return; return;
} }
boolean isBigEnough = Math.abs(element.getBBox().getY1() - element.getBBox().getY2()) > page.getPageHeight() * MIN_TEXTWATERMARK_HEIGHT_THRESHOLD; try (var bbox = element.getBBox()) {
boolean isBigEnough = Math.abs(bbox.getY1() - bbox.getY2()) > page.getPageHeight() * MIN_TEXTWATERMARK_HEIGHT_THRESHOLD;
if (isBigEnough) { if (isBigEnough) {
ElementFeatures elementFeatures = ElementFeatureFactory.extractFeatures(element); ElementFeatures elementFeatures = ElementFeatureFactory.extractFeatures(element);
elementFeaturesLinkedList.add(elementFeatures); elementFeaturesLinkedList.add(elementFeatures);
} }
}
} }
@ -160,9 +164,11 @@ public class WatermarkRemovalService {
@SneakyThrows @SneakyThrows
private boolean isTextRotated(Element element) { private boolean isTextRotated(Element element) {
return Math.abs(element.getCTM().getB()) < Math.sin(Math.toRadians(ROTATED_TEXT_THRESHOLD)) || Math.abs(element.getCTM() try (var ctm = element.getCTM()) {
return Math.abs(ctm.getB()) < Math.sin(Math.toRadians(ROTATED_TEXT_THRESHOLD)) || Math.abs(ctm
.getB()) > Math.sin(Math.toRadians(70 - ROTATED_TEXT_THRESHOLD)); .getB()) > Math.sin(Math.toRadians(70 - ROTATED_TEXT_THRESHOLD));
} }
}
private void shouldTextSearchBeContinued(List<ElementFeatures> elementFeaturesLinkedList) { private void shouldTextSearchBeContinued(List<ElementFeatures> elementFeaturesLinkedList) {
@ -185,7 +191,8 @@ public class WatermarkRemovalService {
if (element.getXObject() == null) { if (element.getXObject() == null) {
return; return;
} }
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringPage && isLocatedNearBorder(element, page)) { try (var bbox = element.getBBox()) {
if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringPage && isLocatedNearBorder(element, page)) {
return; return;
} }
@ -193,16 +200,25 @@ public class WatermarkRemovalService {
ElementFeatures elementFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashOfImage); ElementFeatures elementFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashOfImage);
elementFeaturesLinkedList.add(elementFeatures); elementFeaturesLinkedList.add(elementFeatures);
} }
}
// Typically company logos on dossier pages are located near the border and should be excluded from the watermark removal // Typically company logos on dossier pages are located near the border and should be excluded from the watermark removal
@SneakyThrows @SneakyThrows
private boolean isLocatedNearBorder(Element element, Page page) { private boolean isLocatedNearBorder(Element element, Page page) {
return element.getBBox().getY1() < page.getVisibleContentBox().getY1() + page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || element.getBBox() try (var bbox = element.getBBox(); var contentBox = page.getVisibleContentBox();) {
.getY2() > page.getVisibleContentBox().getY2() - page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || element.getBBox().getX1() < page.getVisibleContentBox() return bbox.getY1() < contentBox.getY1() + page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || bbox
.getX1() + page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD || element.getBBox().getX2() > page.getVisibleContentBox() .getY2() >
.getX2() - page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD; contentBox.getY2() -
page.getPageHeight() *
IMAGE_POSITION_HEIGHT_THRESHOLD ||
bbox.getX1() < contentBox
.getX1() + page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD || bbox.getX2() > contentBox
.getX2() -
page.getPageWidth() *
IMAGE_POSITION_WIDTH_THRESHOLD;
}
} }
@ -214,12 +230,14 @@ public class WatermarkRemovalService {
double minAreaCoveringPage, double minAreaCoveringPage,
Page page) { Page page) {
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringPage) { try (var bbox = element.getBBox()) {
if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringPage) {
return; return;
} }
}
if (visitedXObjIds.add(element.getXObject().getObjNum())) { if (visitedXObjIds.add(element.getXObject().getObjNum())) {
try(ElementReader xObjectReader = new ElementReader()) { try (ElementReader xObjectReader = new ElementReader()) {
xObjectReader.begin(element.getXObject()); xObjectReader.begin(element.getXObject());
for (Element element1 = xObjectReader.next(); element1 != null; element1 = xObjectReader.next()) { for (Element element1 = xObjectReader.next(); element1 != null; element1 = xObjectReader.next()) {
processElement(element1, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page); processElement(element1, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page);
@ -246,7 +264,9 @@ public class WatermarkRemovalService {
.filter(elementFeature -> formObjectsPerPage.values() .filter(elementFeature -> formObjectsPerPage.values()
.stream() .stream()
.filter(elementFeaturesOnPage -> elementFeaturesOnPage.stream() .filter(elementFeaturesOnPage -> elementFeaturesOnPage.stream()
.anyMatch(elementFeature.getElementType() == Element.e_image || elementFeature.getElementType() == Element.e_inline_image ? elementFeature::isSimilarTo : elementFeature::almostMatches)) .anyMatch(
elementFeature.getElementType() == Element.e_image || elementFeature.getElementType() == Element.e_inline_image ?
elementFeature::isSimilarTo : elementFeature::almostMatches))
.count() >= minPagesFilter) .count() >= minPagesFilter)
.toList(); .toList();
} }
@ -255,7 +275,7 @@ public class WatermarkRemovalService {
@SneakyThrows @SneakyThrows
private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) { private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) {
try(ElementReader reader = new ElementReader(); try (ElementReader reader = new ElementReader();
ElementWriter writer = new ElementWriter()) { ElementWriter writer = new ElementWriter()) {
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
@ -295,17 +315,25 @@ public class WatermarkRemovalService {
switch (element.getType()) { switch (element.getType()) {
case Element.e_image, Element.e_inline_image -> { case Element.e_image, Element.e_inline_image -> {
if (element.getBBox() == null) { try (var bbox = element.getBBox()) {
if (bbox == null) {
writer.writeElement(element); writer.writeElement(element);
continue; continue;
} }
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringFromPage && isLocatedNearBorder(element, page) && element.getBBox() }
.getHeight() * element.getBBox().getWidth() < minAreaCoveringFromPage || element.getXObject() == null) { try (var bbox = element.getBBox()) {
if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringFromPage && isLocatedNearBorder(element, page) && bbox
.getHeight() *
bbox
.getWidth() <
minAreaCoveringFromPage ||
element.getXObject() == null) {
writer.writeElement(element); writer.writeElement(element);
continue; continue;
} }
}
removeImages(element, writer, watermarksElementFeaturesList); removeImages(element, writer, watermarksElementFeaturesList);
} }
case Element.e_form -> processForms(page, element, reader, writer, watermarksElementFeaturesList, visitedXObjIds); case Element.e_form -> processForms(page, element, reader, writer, watermarksElementFeaturesList, visitedXObjIds);
@ -344,9 +372,11 @@ public class WatermarkRemovalService {
return false; return false;
} }
if (Math.max(element.getBBox().getY1(), element.getBBox().getY2()) < page.getVisibleContentBox().getY1() + page.getPageHeight() * TEXT_POSITION_THRESHOLD) { try (var bbox = element.getBBox(); var contents = page.getVisibleContentBox();) {
if (Math.max(bbox.getY1(), bbox.getY2()) < contents.getY1() + page.getPageHeight() * TEXT_POSITION_THRESHOLD) {
return false; return false;
} }
}
return true; return true;
} }
@ -385,7 +415,7 @@ public class WatermarkRemovalService {
visitedXObjIds.add(element.getXObject().getObjNum()); visitedXObjIds.add(element.getXObject().getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
reader.formBegin(); reader.formBegin();
formWriter.begin(element.getXObject()); formWriter.begin(element.getXObject());

View File

@ -6,6 +6,7 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -14,6 +15,7 @@ import com.pdftron.pdf.PDFNet;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@SuppressWarnings("PMD") @SuppressWarnings("PMD")
@Slf4j
class InvisibleElementRemovalServiceTest { class InvisibleElementRemovalServiceTest {
InvisibleElementRemovalService invisibleElementRemovalService; InvisibleElementRemovalService invisibleElementRemovalService;
@ -101,8 +103,19 @@ class InvisibleElementRemovalServiceTest {
String result = PdfTextExtraction.extractAllTextFromDocument(in); String result = PdfTextExtraction.extractAllTextFromDocument(in);
assertThat(result).contains("#1 Dark", assertThat(result).contains("#1 Dark",
"#13 Yellow", "#13 Yellow",
"Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip\n" + "ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie\n" + "consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim\n" + "qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi."); "Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip\n" +
assertThat(result).doesNotContain("Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut\n" + "labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et\n" + "ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem\n" + "ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et\n" + "dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea\n" + "rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum\n" + "dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore\n" + "magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n" + "clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. "); "ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie\n" +
"consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim\n" +
"qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.");
assertThat(result).doesNotContain("Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut\n" +
"labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et\n" +
"ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem\n" +
"ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et\n" +
"dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea\n" +
"rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum\n" +
"dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore\n" +
"magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n" +
"clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. ");
} }
} }