Marmelator/pdftron-logic-commons
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

memory optimisations

Browse Source
This commit is contained in:
Timo Bejan 2024-03-14 18:50:07 +02:00
parent e926083881
commit 1b4ab8dc88
6 changed files with 283 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/main/java/com/iqser/red/pdftronlogic/commons/ClippingPathStack.java
View File

@ -1,7 +1,5 @@
package com.iqser.red.pdftronlogic.commons; package com.iqser.red.pdftronlogic.commons;
import static com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService.TOLERANCE; import static com.iqser.red.pdftronlogic.commons.InvisibleElementRemovalService.TOLERANCE;
import java.awt.geom.Area; import java.awt.geom.Area;
@ -35,6 +33,13 @@ public class ClippingPathStack {
} }
@SneakyThrows
public void intersectClippingPath(Rectangle2D path) {
getCurrentClippingPath().intersect(new Area(path));
}
public boolean almostIntersects(double x, double y, double width, double height) { public boolean almostIntersects(double x, double y, double width, double height) {
// To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle // To address inconsistencies in the calculation of the bounding box we slightly increase the rectangle
// Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0. // Height or width are zero for straight lines, even though they are being rendered. Therefore, height or width must be at minimum >0.
@ -56,15 +61,16 @@ public class ClippingPathStack {
public void enterNewGState() { public void enterNewGState() {
Area current = stack.peek(); Area current = stack.peek();
Area cloned = new Area(); Area cloned = (Area) current.clone();
cloned.add(current);
stack.push(cloned); stack.push(cloned);
} }
public void leaveGState() { public void leaveGState() {
stack.pop(); // somehow this greatly helps memory management
var popped = stack.pop();
popped.reset();
} }
} }

82
src/main/java/com/iqser/red/pdftronlogic/commons/ElementFeatureFactory.java
View File

@ -4,6 +4,7 @@ import com.pdftron.common.PDFNetException;
import com.pdftron.pdf.Element; import com.pdftron.pdf.Element;
public class ElementFeatureFactory { public class ElementFeatureFactory {
public static ElementFeatures extractFeatures(Element element) throws PDFNetException { public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
return switch (element.getType()) { return switch (element.getType()) {
@ -16,7 +17,9 @@ public class ElementFeatureFactory {
}; };
} }
public static ElementFeatures extractFeaturesWithHash(Element element, String hashObject) throws PDFNetException { public static ElementFeatures extractFeaturesWithHash(Element element, String hashObject) throws PDFNetException {
return buildImage(element) return buildImage(element)
.hashOfImage(hashObject) .hashOfImage(hashObject)
.build(); .build();
@ -25,56 +28,65 @@ public class ElementFeatureFactory {
private static ElementFeatures.Form buildForm(Element element) throws PDFNetException { private static ElementFeatures.Form buildForm(Element element) throws PDFNetException {
return ElementFeatures.Form.builder() try (var bbox = element.getBBox();) {
.elementType(element.getType()) return ElementFeatures.Form.builder()
.boundingBox(Converter.toRectangle2D(element.getBBox())) .elementType(element.getType())
.xObjectType(element.getXObject().getType()) .boundingBox(Converter.toRectangle2D(bbox))
.dictOrArrayOrStreamLength(element.getXObject().getType() == 7 ? element.getXObject().getDecodedStream().size() : 0) .xObjectType(element.getXObject().getType())
.build(); .dictOrArrayOrStreamLength(element.getXObject().getType() == 7 ? element.getXObject().getDecodedStream().size() : 0)
.build();
}
} }
private static ElementFeatures.Image.ImageBuilder<?, ?> buildImage(Element element) throws PDFNetException { private static ElementFeatures.Image.ImageBuilder<?, ?> buildImage(Element element) throws PDFNetException {
return ElementFeatures.Image.builder() try (var bbox = element.getBBox();) {
.elementType(element.getType()) return ElementFeatures.Image.builder()
.boundingBox(Converter.toRectangle2D(element.getBBox())) .elementType(element.getType())
.dataSize(element.getImageDataSize()) .boundingBox(Converter.toRectangle2D(bbox))
.height(element.getImageHeight()) .dataSize(element.getImageDataSize())
.width(element.getImageWidth()) .height(element.getImageHeight())
.renderingIntent(element.getImageRenderingIntent()) .width(element.getImageWidth())
.componentNum(element.getComponentNum()) .renderingIntent(element.getImageRenderingIntent())
.bitsPerComponent(element.getBitsPerComponent()); .componentNum(element.getComponentNum())
.bitsPerComponent(element.getBitsPerComponent());
}
} }
private static ElementFeatures.Text buildText(Element element) throws PDFNetException { private static ElementFeatures.Text buildText(Element element) throws PDFNetException {
return ElementFeatures.Text.builder() try (var bbox = element.getBBox();) {
.elementType(element.getType()) return ElementFeatures.Text.builder()
.boundingBox(Converter.toRectangle2D(element.getBBox())) .elementType(element.getType())
.text(element.getTextString()) .boundingBox(Converter.toRectangle2D(bbox))
.font(element.getGState().getFont().getType()) .text(element.getTextString())
.fontsize(element.getGState().getFontSize()) .font(element.getGState().getFont().getType())
.build(); .fontsize(element.getGState().getFontSize())
.build();
}
} }
private static ElementFeatures.Path buildPath(Element element) throws PDFNetException { private static ElementFeatures.Path buildPath(Element element) throws PDFNetException {
return ElementFeatures.Path.builder() try (var bbox = element.getBBox(); var ctm = element.getCTM();
.elementType(element.getType()) var fillColor = element.getGState().getFillColor();
.boundingBox(Converter.toRectangle2D(element.getBBox())) var strokeColor = element.getGState().getStrokeColor()) {
.isClippingPath(element.isClippingPath()) return ElementFeatures.Path.builder()
.isClipWindingFill(element.isClipWindingFill()) .elementType(element.getType())
.isStroked(element.isStroked()) .boundingBox(Converter.toRectangle2D(bbox))
.isFilled(element.isFilled()) .isClippingPath(element.isClippingPath())
.isWindingFill(element.isWindingFill()) .isClipWindingFill(element.isClipWindingFill())
.fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), element.getGState().getFillColor())) .isStroked(element.isStroked())
.strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), element.getGState().getStrokeColor())) .isFilled(element.isFilled())
.linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), element.getCTM())) .isWindingFill(element.isWindingFill())
.build(); .fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), fillColor))
.strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), strokeColor))
.linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), ctm))
.build();
}
} }
} }

85
src/main/java/com/iqser/red/pdftronlogic/commons/ElementFeatures.java
View File

@ -31,9 +31,11 @@ public class ElementFeatures {
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
return element.getType() == elementType && // try (var bbox = element.getBBox()) {
element.getBBox() != null && // return element.getType() == elementType && //
rectsAlmostMatch(element.getBBox()); bbox != null && //
rectsAlmostMatch(bbox);
}
} }
@ -42,9 +44,9 @@ public class ElementFeatures {
// To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance // To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
return almostEqual(bBox.getX1(), boundingBox.getX()) && // return almostEqual(bBox.getX1(), boundingBox.getX()) && //
almostEqual(bBox.getY1(), boundingBox.getY()) && // almostEqual(bBox.getY1(), boundingBox.getY()) && //
almostEqual(bBox.getWidth(), boundingBox.getWidth()) && // almostEqual(bBox.getWidth(), boundingBox.getWidth()) && //
almostEqual(bBox.getHeight(), boundingBox.getHeight()); almostEqual(bBox.getHeight(), boundingBox.getHeight());
} }
@ -65,9 +67,9 @@ public class ElementFeatures {
// To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance // To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
return almostEqual(bBox.getX(), boundingBox.getX()) && // return almostEqual(bBox.getX(), boundingBox.getX()) && //
almostEqual(bBox.getY(), boundingBox.getY()) && // almostEqual(bBox.getY(), boundingBox.getY()) && //
almostEqual(bBox.getWidth(), boundingBox.getWidth()) && // almostEqual(bBox.getWidth(), boundingBox.getWidth()) && //
almostEqual(bBox.getHeight(), boundingBox.getHeight()); almostEqual(bBox.getHeight(), boundingBox.getHeight());
} }
@ -81,9 +83,9 @@ public class ElementFeatures {
// To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance // To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
return isPositionSimilar(rectangle2D.getX(), boundingBox.getX(), rectangle2D.getWidth()) && // return isPositionSimilar(rectangle2D.getX(), boundingBox.getX(), rectangle2D.getWidth()) && //
isPositionSimilar(rectangle2D.getY(), boundingBox.getY(), rectangle2D.getHeight()) && // isPositionSimilar(rectangle2D.getY(), boundingBox.getY(), rectangle2D.getHeight()) && //
isSizeSimilar(rectangle2D.getWidth(), boundingBox.getWidth()) && // isSizeSimilar(rectangle2D.getWidth(), boundingBox.getWidth()) && //
isSizeSimilar(rectangle2D.getHeight(), boundingBox.getHeight()); isSizeSimilar(rectangle2D.getHeight(), boundingBox.getHeight());
} }
@ -115,9 +117,9 @@ public class ElementFeatures {
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
return super.almostMatches(element) && // return super.almostMatches(element) && //
text.equals(element.getTextString()) && // text.equals(element.getTextString()) && //
font == element.getGState().getFont().getType() && // font == element.getGState().getFont().getType() && //
almostEqual(fontsize, element.getGState().getFontSize()); almostEqual(fontsize, element.getGState().getFontSize());
} }
} }
@ -142,11 +144,11 @@ public class ElementFeatures {
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
return super.almostMatches(element) && // return super.almostMatches(element) && //
isClippingPath == element.isClippingPath() && // isClippingPath == element.isClippingPath() && //
isClipWindingFill == element.isClipWindingFill() && // isClipWindingFill == element.isClipWindingFill() && //
isStroked == element.isStroked() && // isStroked == element.isStroked() && //
isFilled == element.isFilled() && // isFilled == element.isFilled() && //
isWindingFill == element.isWindingFill(); isWindingFill == element.isWindingFill();
} }
@ -161,7 +163,7 @@ public class ElementFeatures {
public boolean isBackground(Rect area) { public boolean isBackground(Rect area) {
return isFilled && // return isFilled && //
getBoundingBox().intersects(area.getX1(), area.getY1(), area.getWidth(), area.getHeight()); getBoundingBox().intersects(area.getX1(), area.getY1(), area.getWidth(), area.getHeight());
} }
} }
@ -185,12 +187,12 @@ public class ElementFeatures {
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
return super.almostMatches(element) && // return super.almostMatches(element) && //
dataSize == element.getImageDataSize() && // dataSize == element.getImageDataSize() && //
height == element.getImageHeight() && // height == element.getImageHeight() && //
width == element.getImageWidth() && // width == element.getImageWidth() && //
renderingIntent == element.getImageRenderingIntent() && // renderingIntent == element.getImageRenderingIntent() && //
componentNum == element.getComponentNum() && // componentNum == element.getComponentNum() && //
bitsPerComponent == element.getBitsPerComponent(); bitsPerComponent == element.getBitsPerComponent();
} }
@ -199,15 +201,18 @@ public class ElementFeatures {
if (elementFeatures.getClass() != this.getClass()) { if (elementFeatures.getClass() != this.getClass()) {
return false; return false;
} }
return super.almostMatches(elementFeatures) && this.dataSize == ((Image) elementFeatures).getDataSize() && this.height == ((Image) elementFeatures).getHeight() && this.width == ((Image) elementFeatures).getWidth() && this.renderingIntent == ((Image) elementFeatures).getRenderingIntent() && this.componentNum == ((Image) elementFeatures).getComponentNum() && this.bitsPerComponent == ((Image) elementFeatures).getBitsPerComponent() && calculateHammingDistance( return super.almostMatches(elementFeatures) && this.dataSize == ((Image) elementFeatures).getDataSize() && this.height == ((Image) elementFeatures).getHeight() &&
((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD; this.width == ((Image) elementFeatures).getWidth() && this.renderingIntent == ((Image) elementFeatures).getRenderingIntent() &&
this.componentNum == ((Image) elementFeatures).getComponentNum() && this.bitsPerComponent == ((Image) elementFeatures).getBitsPerComponent() &&
calculateHammingDistance(
((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD;
} }
public boolean isSimilarTo(ElementFeatures elementFeatures) { public boolean isSimilarTo(ElementFeatures elementFeatures) {
return super.isSimilarTo(elementFeatures) && // return super.isSimilarTo(elementFeatures) && //
calculateHammingDistance(((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD; calculateHammingDistance(((Image) elementFeatures).getHashOfImage()) <= HAMMING_DISTANCE_THRESHOLD;
} }
@ -241,10 +246,12 @@ public class ElementFeatures {
@Override @Override
public boolean almostMatches(Element element) throws PDFNetException { public boolean almostMatches(Element element) throws PDFNetException {
return element.getType() == getElementType() && // try (var bbox = element.getBBox()) {
element.getBBox() != null && // return element.getType() == getElementType() && //
(super.rectsAlmostMatch(element.getBBox()) || almostRotateMatches(element.getBBox().getRectangle())) && xObjectType == element.getXObject() bbox != null && //
.getType() && dictOrArrayOrStreamLength == element.getXObject().getDecodedStream().size(); (super.rectsAlmostMatch(bbox) || almostRotateMatches(bbox.getRectangle())) && xObjectType == element.getXObject()
.getType() && dictOrArrayOrStreamLength == element.getXObject().getDecodedStream().size();
}
} }
@ -253,9 +260,11 @@ public class ElementFeatures {
if (elementFeatures.getClass() != this.getClass()) { if (elementFeatures.getClass() != this.getClass()) {
return false; return false;
} }
return elementFeatures.getElementType() == getElementType() && elementFeatures.getBoundingBox() != null && (super.rectsAlmostMatch(elementFeatures.getBoundingBox()) || almostRotateMatches( return elementFeatures.getElementType() == getElementType() && elementFeatures.getBoundingBox() != null &&
elementFeatures.getBoundingBox() (super.rectsAlmostMatch(elementFeatures.getBoundingBox()) || almostRotateMatches(
.getBounds2D())) && xObjectType == ((Form) elementFeatures).getXObjectType() && dictOrArrayOrStreamLength == ((Form) elementFeatures).getDictOrArrayOrStreamLength(); elementFeatures.getBoundingBox()
.getBounds2D())) && xObjectType == ((Form) elementFeatures).getXObjectType() &&
dictOrArrayOrStreamLength == ((Form) elementFeatures).getDictOrArrayOrStreamLength();
} }
@ -263,7 +272,7 @@ public class ElementFeatures {
private boolean almostRotateMatches(Rectangle2D bBox) { private boolean almostRotateMatches(Rectangle2D bBox) {
return almostEqual(bBox.getWidth(), getBoundingBox().getHeight()) && // return almostEqual(bBox.getWidth(), getBoundingBox().getHeight()) && //
almostEqual(bBox.getHeight(), getBoundingBox().getWidth()); almostEqual(bBox.getHeight(), getBoundingBox().getWidth());
} }
} }

162
src/main/java/com/iqser/red/pdftronlogic/commons/InvisibleElementRemovalService.java
View File

@ -63,15 +63,12 @@ public class InvisibleElementRemovalService {
PDFDoc pdfDoc = new PDFDoc(pdfFile); PDFDoc pdfDoc = new PDFDoc(pdfFile);
execute(pdfDoc, delta, removePaths, markedContentToIgnore); try (pdfDoc) {
execute(pdfDoc, delta, removePaths, markedContentToIgnore);
try {
pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null); pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
} catch (Exception e) { } catch (Exception e) {
log.error("File could not be saved after invisible element removal"); log.error("File could not be saved after invisible element removal");
throw new RuntimeException(e); throw new RuntimeException(e);
} finally {
pdfDoc.close();
} }
} }
@ -151,8 +148,8 @@ public class InvisibleElementRemovalService {
private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) { private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) {
log.info("Start removing invisible Elements"); log.info("Start removing invisible Elements");
try(ElementWriter writer = new ElementWriter(); try (ElementWriter writer = new ElementWriter();
ElementReader reader = new ElementReader()) { ElementReader reader = new ElementReader()) {
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) { for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
@ -236,7 +233,7 @@ public class InvisibleElementRemovalService {
private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException { private void processImages(Element imageElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
try(Rect rect = imageElement.getBBox()) { try (Rect rect = imageElement.getBBox()) {
if (rect == null) { if (rect == null) {
return; return;
@ -257,7 +254,7 @@ public class InvisibleElementRemovalService {
private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException { private void processText(Element textElement, ElementWriter writer, InvisibleElementRemovalContext context) throws PDFNetException {
try(Rect textBBox = textElement.getBBox()) { try (Rect textBBox = textElement.getBBox()) {
if (textBBox == null) { if (textBBox == null) {
writer.writeElement(textElement); writer.writeElement(textElement);
@ -289,13 +286,17 @@ public class InvisibleElementRemovalService {
if (!inClippingPath) { if (!inClippingPath) {
gState.setFillColorSpace(ColorSpace.createDeviceRGB()); gState.setFillColorSpace(ColorSpace.createDeviceRGB());
// red for elements removed by clipping path // red for elements removed by clipping path
gState.setFillColor(new ColorPt(1, 0, 0)); try (var color = new ColorPt(1, 0, 0)) {
gState.setFillColor(color);
}
writer.writeElement(textElement); writer.writeElement(textElement);
} }
if (!isTextVisible) { if (!isTextVisible) {
gState.setFillColorSpace(ColorSpace.createDeviceRGB()); gState.setFillColorSpace(ColorSpace.createDeviceRGB());
// blue for elements removed due to transparency or not rendered or same color as background // blue for elements removed due to transparency or not rendered or same color as background
gState.setFillColor(new ColorPt(0, 0, 1)); try (var color = new ColorPt(0, 0, 1)) {
gState.setFillColor(color);
}
gState.setTextRenderMode(GState.e_fill_text); gState.setTextRenderMode(GState.e_fill_text);
gState.setFillOpacity(1); gState.setFillOpacity(1);
writer.writeElement(textElement); writer.writeElement(textElement);
@ -314,19 +315,21 @@ public class InvisibleElementRemovalService {
context.visitedXObjIds().add(formObj.getObjNum()); context.visitedXObjIds().add(formObj.getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
context.clippingPathStack().enterNewGState(); context.clippingPathStack().enterNewGState();
context.clippingPathStack().intersectClippingPath(new GeneralPath(Converter.toRectangle2D(formElement.getBBox()))); try (var formElementBBOX = formElement.getBBox()) {
context.reader().formBegin(); context.clippingPathStack().intersectClippingPath(Converter.toRectangle2D(formElementBBOX));
formWriter.begin(formObj); context.reader().formBegin();
formWriter.begin(formObj);
context.reader().clearChangeList(); context.reader().clearChangeList();
formWriter.setDefaultGState(context.reader()); formWriter.setDefaultGState(context.reader());
processElements(formWriter, context); processElements(formWriter, context);
formWriter.end(); formWriter.end();
context.reader().end(); context.reader().end();
context.clippingPathStack().leaveGState(); context.clippingPathStack().leaveGState();
}
} }
} }
} }
@ -336,52 +339,58 @@ public class InvisibleElementRemovalService {
PathData pathData = pathElement.getPathData(); PathData pathData = pathElement.getPathData();
if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0 || pathElement.getBBox() == null) { try (var bbox = pathElement.getBBox()) {
writer.writeElement(pathElement); if (pathData.getOperators().length == 0 && pathData.getPoints().length == 0 || bbox == null) {
return; writer.writeElement(pathElement);
} return;
GeneralPath linePath = Converter.convertToGeneralPathAndTransformToInitialUserSpace(pathData, pathElement.getCTM());
var rect = linePath.getBounds2D();
boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX(), rect.getY(), rect.getWidth(), rect.getHeight());
if (pathElement.isClippingPath()) {
if (pathElement.isClipWindingFill()) {
linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
} else {
linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
} }
context.clippingPathStack().intersectClippingPath(linePath); try (var ctm = pathElement.getCTM()) {
pathElement.setPathClip(!context.delta()); GeneralPath linePath = Converter.convertToGeneralPathAndTransformToInitialUserSpace(pathData, ctm);
writer.writeElement(pathElement);
} else { var rect = linePath.getBounds2D();
if (pathElement.isWindingFill()) {
linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
} else {
linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
}
if (inClippingPath) { boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX(), rect.getY(), rect.getWidth(), rect.getHeight());
if (isFilledAndNonTransparent(pathElement)) {
calculateOverlapsForLinePath(context, linePath); if (pathElement.isClippingPath()) {
if (pathElement.isClipWindingFill()) {
linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
} else {
linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
}
context.clippingPathStack().intersectClippingPath(linePath);
pathElement.setPathClip(!context.delta());
writer.writeElement(pathElement);
} else {
if (pathElement.isWindingFill()) {
linePath.setWindingRule(GeneralPath.WIND_NON_ZERO);
} else {
linePath.setWindingRule(GeneralPath.WIND_EVEN_ODD);
}
if (inClippingPath) {
if (isFilledAndNonTransparent(pathElement)) {
calculateOverlapsForLinePath(context, linePath);
}
context.visibleElements().add(ElementFeatureFactory.extractFeatures(pathElement));
}
if (!context.delta() && (inClippingPath || !context.removePaths())) {
writer.writeElement(pathElement);
}
if (context.delta() && !inClippingPath && context.removePaths()) {
try (var color = new ColorPt(1, 0, 0)) {
pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setFillColor(color);
pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setStrokeColor(color);
writer.writeElement(pathElement);
}
}
} }
context.visibleElements().add(ElementFeatureFactory.extractFeatures(pathElement));
}
if (!context.delta() && (inClippingPath || !context.removePaths())) {
writer.writeElement(pathElement);
}
if (context.delta() && !inClippingPath && context.removePaths()) {
pathElement.getGState().setFillColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setFillColor(new ColorPt(1, 0, 0));
pathElement.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
pathElement.getGState().setStrokeColor(new ColorPt(1, 0, 0));
writer.writeElement(pathElement);
} }
} }
} }
@ -480,7 +489,7 @@ public class InvisibleElementRemovalService {
context.visitedXObjIds().add(formObj.getObjNum()); context.visitedXObjIds().add(formObj.getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
context.reader().formBegin(); context.reader().formBegin();
formWriter.begin(formObj); formWriter.begin(formObj);
@ -516,7 +525,9 @@ public class InvisibleElementRemovalService {
private boolean fillIsVisible(GState gState, Rect textBBox, InvisibleElementRemovalContext context) throws PDFNetException { private boolean fillIsVisible(GState gState, Rect textBBox, InvisibleElementRemovalContext context) throws PDFNetException {
return gState.getFillOpacity() != 0 && differentColorThanBackgroundColor(Converter.convertColor(gState.getFillColorSpace(), gState.getFillColor()), textBBox, context); try (var color = gState.getFillColor()) {
return gState.getFillOpacity() != 0 && differentColorThanBackgroundColor(Converter.convertColor(gState.getFillColorSpace(), color), textBBox, context);
}
} }
@ -541,13 +552,16 @@ public class InvisibleElementRemovalService {
private static List<ElementFeatures.Path> findVisiblePathElementsThatIntersect(Rect textBBox, InvisibleElementRemovalContext context) { private static List<ElementFeatures.Path> findVisiblePathElementsThatIntersect(Rect textBBox, InvisibleElementRemovalContext context) {
return context.visibleElements() var result = new ArrayList<ElementFeatures.Path>();
.stream() for (var element : context.visibleElements()) {
.filter(elementFeatures -> elementFeatures.getElementType() == Element.e_path) if (element.getElementType() == Element.e_path
.map(elementFeatures -> (ElementFeatures.Path) elementFeatures) && !((ElementFeatures.Path) element).getFillColor().equals(Color.WHITE)
.filter(elementFeatures -> !elementFeatures.getFillColor().equals(Color.WHITE)) && ((ElementFeatures.Path) element).isBackground(textBBox)) {
.filter(element -> element.isBackground(textBBox)) result.add((ElementFeatures.Path) element);
.toList(); }
}
return result;
} }
@ -581,10 +595,10 @@ public class InvisibleElementRemovalService {
@SneakyThrows @SneakyThrows
private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) { private void drawBBox(ElementWriter writer, Rectangle2D r, String hexcolor) {
try(ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d, try (ColorPt colorPt = new ColorPt(Integer.valueOf(hexcolor.substring(1, 3), 16) / 255d,
Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d, Integer.valueOf(hexcolor.substring(3, 5), 16) / 255d,
Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d); Integer.valueOf(hexcolor.substring(5, 7), 16) / 255d);
ElementBuilder eb = new ElementBuilder()) { ElementBuilder eb = new ElementBuilder()) {
Element rect = eb.createRect(r.getX(), r.getY(), r.getWidth(), r.getHeight()); Element rect = eb.createRect(r.getX(), r.getY(), r.getWidth(), r.getHeight());
rect.setPathStroke(true); rect.setPathStroke(true);
rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB()); rect.getGState().setStrokeColorSpace(ColorSpace.createDeviceRGB());
@ -608,4 +622,4 @@ public class InvisibleElementRemovalService {
} }
} }

116
src/main/java/com/iqser/red/pdftronlogic/commons/WatermarkRemovalService.java
View File

@ -93,7 +93,7 @@ public class WatermarkRemovalService {
Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>(); Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>();
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
try(ElementReader reader = new ElementReader()) { try (ElementReader reader = new ElementReader()) {
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) { for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
@ -123,14 +123,16 @@ public class WatermarkRemovalService {
double minAreaCoveringPage, double minAreaCoveringPage,
Page page) throws PDFNetException { Page page) throws PDFNetException {
if (element.getBBox() == null) { try (var bbox = element.getBBox()) {
return; if (bbox == null) {
} return;
}
switch (element.getType()) { switch (element.getType()) {
case Element.e_form -> processXObject(element, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page); case Element.e_form -> processXObject(element, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page);
case Element.e_image, Element.e_inline_image -> processImages(element, elementFeaturesLinkedList, page, minAreaCoveringPage); case Element.e_image, Element.e_inline_image -> processImages(element, elementFeaturesLinkedList, page, minAreaCoveringPage);
case Element.e_text -> processText(element, elementFeaturesLinkedList, page); case Element.e_text -> processText(element, elementFeaturesLinkedList, page);
}
} }
} }
@ -147,11 +149,13 @@ public class WatermarkRemovalService {
return; return;
} }
boolean isBigEnough = Math.abs(element.getBBox().getY1() - element.getBBox().getY2()) > page.getPageHeight() * MIN_TEXTWATERMARK_HEIGHT_THRESHOLD; try (var bbox = element.getBBox()) {
boolean isBigEnough = Math.abs(bbox.getY1() - bbox.getY2()) > page.getPageHeight() * MIN_TEXTWATERMARK_HEIGHT_THRESHOLD;
if (isBigEnough) { if (isBigEnough) {
ElementFeatures elementFeatures = ElementFeatureFactory.extractFeatures(element); ElementFeatures elementFeatures = ElementFeatureFactory.extractFeatures(element);
elementFeaturesLinkedList.add(elementFeatures); elementFeaturesLinkedList.add(elementFeatures);
}
} }
} }
@ -160,8 +164,10 @@ public class WatermarkRemovalService {
@SneakyThrows @SneakyThrows
private boolean isTextRotated(Element element) { private boolean isTextRotated(Element element) {
return Math.abs(element.getCTM().getB()) < Math.sin(Math.toRadians(ROTATED_TEXT_THRESHOLD)) || Math.abs(element.getCTM() try (var ctm = element.getCTM()) {
.getB()) > Math.sin(Math.toRadians(70 - ROTATED_TEXT_THRESHOLD)); return Math.abs(ctm.getB()) < Math.sin(Math.toRadians(ROTATED_TEXT_THRESHOLD)) || Math.abs(ctm
.getB()) > Math.sin(Math.toRadians(70 - ROTATED_TEXT_THRESHOLD));
}
} }
@ -185,13 +191,15 @@ public class WatermarkRemovalService {
if (element.getXObject() == null) { if (element.getXObject() == null) {
return; return;
} }
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringPage && isLocatedNearBorder(element, page)) { try (var bbox = element.getBBox()) {
return; if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringPage && isLocatedNearBorder(element, page)) {
} return;
}
String hashOfImage = ImageHashFactory.calculate(element); String hashOfImage = ImageHashFactory.calculate(element);
ElementFeatures elementFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashOfImage); ElementFeatures elementFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashOfImage);
elementFeaturesLinkedList.add(elementFeatures); elementFeaturesLinkedList.add(elementFeatures);
}
} }
@ -199,10 +207,18 @@ public class WatermarkRemovalService {
@SneakyThrows @SneakyThrows
private boolean isLocatedNearBorder(Element element, Page page) { private boolean isLocatedNearBorder(Element element, Page page) {
return element.getBBox().getY1() < page.getVisibleContentBox().getY1() + page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || element.getBBox() try (var bbox = element.getBBox(); var contentBox = page.getVisibleContentBox();) {
.getY2() > page.getVisibleContentBox().getY2() - page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || element.getBBox().getX1() < page.getVisibleContentBox() return bbox.getY1() < contentBox.getY1() + page.getPageHeight() * IMAGE_POSITION_HEIGHT_THRESHOLD || bbox
.getX1() + page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD || element.getBBox().getX2() > page.getVisibleContentBox() .getY2() >
.getX2() - page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD; contentBox.getY2() -
page.getPageHeight() *
IMAGE_POSITION_HEIGHT_THRESHOLD ||
bbox.getX1() < contentBox
.getX1() + page.getPageWidth() * IMAGE_POSITION_WIDTH_THRESHOLD || bbox.getX2() > contentBox
.getX2() -
page.getPageWidth() *
IMAGE_POSITION_WIDTH_THRESHOLD;
}
} }
@ -214,12 +230,14 @@ public class WatermarkRemovalService {
double minAreaCoveringPage, double minAreaCoveringPage,
Page page) { Page page) {
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringPage) { try (var bbox = element.getBBox()) {
return; if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringPage) {
return;
}
} }
if (visitedXObjIds.add(element.getXObject().getObjNum())) { if (visitedXObjIds.add(element.getXObject().getObjNum())) {
try(ElementReader xObjectReader = new ElementReader()) { try (ElementReader xObjectReader = new ElementReader()) {
xObjectReader.begin(element.getXObject()); xObjectReader.begin(element.getXObject());
for (Element element1 = xObjectReader.next(); element1 != null; element1 = xObjectReader.next()) { for (Element element1 = xObjectReader.next(); element1 != null; element1 = xObjectReader.next()) {
processElement(element1, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page); processElement(element1, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage, page);
@ -244,10 +262,12 @@ public class WatermarkRemovalService {
.stream() .stream()
.flatMap(Collection::stream) .flatMap(Collection::stream)
.filter(elementFeature -> formObjectsPerPage.values() .filter(elementFeature -> formObjectsPerPage.values()
.stream() .stream()
.filter(elementFeaturesOnPage -> elementFeaturesOnPage.stream() .filter(elementFeaturesOnPage -> elementFeaturesOnPage.stream()
.anyMatch(elementFeature.getElementType() == Element.e_image || elementFeature.getElementType() == Element.e_inline_image ? elementFeature::isSimilarTo : elementFeature::almostMatches)) .anyMatch(
.count() >= minPagesFilter) elementFeature.getElementType() == Element.e_image || elementFeature.getElementType() == Element.e_inline_image ?
elementFeature::isSimilarTo : elementFeature::almostMatches))
.count() >= minPagesFilter)
.toList(); .toList();
} }
@ -255,8 +275,8 @@ public class WatermarkRemovalService {
@SneakyThrows @SneakyThrows
private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) { private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) {
try(ElementReader reader = new ElementReader(); try (ElementReader reader = new ElementReader();
ElementWriter writer = new ElementWriter()) { ElementWriter writer = new ElementWriter()) {
Set<Long> visitedXObjIds = new TreeSet<>(); Set<Long> visitedXObjIds = new TreeSet<>();
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) { for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
@ -295,16 +315,24 @@ public class WatermarkRemovalService {
switch (element.getType()) { switch (element.getType()) {
case Element.e_image, Element.e_inline_image -> { case Element.e_image, Element.e_inline_image -> {
if (element.getBBox() == null) { try (var bbox = element.getBBox()) {
writer.writeElement(element); if (bbox == null) {
continue; writer.writeElement(element);
continue;
}
} }
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringFromPage && isLocatedNearBorder(element, page) && element.getBBox() try (var bbox = element.getBBox()) {
.getHeight() * element.getBBox().getWidth() < minAreaCoveringFromPage || element.getXObject() == null) { if (bbox.getHeight() * bbox.getWidth() < minAreaCoveringFromPage && isLocatedNearBorder(element, page) && bbox
.getHeight() *
bbox
.getWidth() <
minAreaCoveringFromPage ||
element.getXObject() == null) {
writer.writeElement(element); writer.writeElement(element);
continue; continue;
}
} }
removeImages(element, writer, watermarksElementFeaturesList); removeImages(element, writer, watermarksElementFeaturesList);
} }
@ -344,8 +372,10 @@ public class WatermarkRemovalService {
return false; return false;
} }
if (Math.max(element.getBBox().getY1(), element.getBBox().getY2()) < page.getVisibleContentBox().getY1() + page.getPageHeight() * TEXT_POSITION_THRESHOLD) { try (var bbox = element.getBBox(); var contents = page.getVisibleContentBox();) {
return false; if (Math.max(bbox.getY1(), bbox.getY2()) < contents.getY1() + page.getPageHeight() * TEXT_POSITION_THRESHOLD) {
return false;
}
} }
return true; return true;
} }
@ -385,7 +415,7 @@ public class WatermarkRemovalService {
visitedXObjIds.add(element.getXObject().getObjNum()); visitedXObjIds.add(element.getXObject().getObjNum());
// writer needs to be newly initialized when entering a new content stream // writer needs to be newly initialized when entering a new content stream
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest) // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
try(ElementWriter formWriter = new ElementWriter()) { try (ElementWriter formWriter = new ElementWriter()) {
reader.formBegin(); reader.formBegin();
formWriter.begin(element.getXObject()); formWriter.begin(element.getXObject());

19
src/test/java/com/iqser/red/pdftronlogic/commons/InvisibleElementRemovalServiceTest.java
View File

@ -6,6 +6,7 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileOutputStream; import java.io.FileOutputStream;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
@ -14,6 +15,7 @@ import com.pdftron.pdf.PDFNet;
import lombok.SneakyThrows; import lombok.SneakyThrows;
@SuppressWarnings("PMD") @SuppressWarnings("PMD")
@Slf4j
class InvisibleElementRemovalServiceTest { class InvisibleElementRemovalServiceTest {
InvisibleElementRemovalService invisibleElementRemovalService; InvisibleElementRemovalService invisibleElementRemovalService;
@ -101,8 +103,19 @@ class InvisibleElementRemovalServiceTest {
String result = PdfTextExtraction.extractAllTextFromDocument(in); String result = PdfTextExtraction.extractAllTextFromDocument(in);
assertThat(result).contains("#1 Dark", assertThat(result).contains("#1 Dark",
"#13 Yellow", "#13 Yellow",
"Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip\n" + "ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie\n" + "consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim\n" + "qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi."); "Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip\n" +
assertThat(result).doesNotContain("Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut\n" + "labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et\n" + "ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem\n" + "ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et\n" + "dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea\n" + "rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum\n" + "dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore\n" + "magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n" + "clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. "); "ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie\n" +
"consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim\n" +
"qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi.");
assertThat(result).doesNotContain("Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut\n" +
"labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et\n" +
"ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem\n" +
"ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et\n" +
"dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea\n" +
"rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum\n" +
"dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore\n" +
"magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet\n" +
"clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. ");
} }
} }
@ -176,4 +189,4 @@ class InvisibleElementRemovalServiceTest {
} }
} }