Merge branch 'RED-7075' into 'master'
Resolve RED-7075 Closes RED-7075 See merge request redactmanager/commons/pdftron-logic-commons!11
This commit is contained in:
commit
35fcee4824
6
pom.xml
6
pom.xml
@ -26,6 +26,12 @@
|
|||||||
<artifactId>slf4j-api</artifactId>
|
<artifactId>slf4j-api</artifactId>
|
||||||
<scope>provided</scope>
|
<scope>provided</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.logging.log4j</groupId>
|
||||||
|
<artifactId>log4j-slf4j2-impl</artifactId>
|
||||||
|
<version>2.20.0</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.google.guava</groupId>
|
<groupId>com.google.guava</groupId>
|
||||||
<artifactId>guava</artifactId>
|
<artifactId>guava</artifactId>
|
||||||
|
|||||||
@ -0,0 +1,80 @@
|
|||||||
|
package com.iqser.red.pdftronlogic.commons;
|
||||||
|
|
||||||
|
import com.pdftron.common.PDFNetException;
|
||||||
|
import com.pdftron.pdf.Element;
|
||||||
|
|
||||||
|
public class ElementFeatureFactory {
|
||||||
|
public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
|
||||||
|
|
||||||
|
return switch (element.getType()) {
|
||||||
|
case Element.e_path -> buildPath(element);
|
||||||
|
case Element.e_text -> buildText(element);
|
||||||
|
case Element.e_image, Element.e_inline_image -> buildImage(element).build();
|
||||||
|
case Element.e_form -> buildForm(element);
|
||||||
|
// This technically should never happen, it's a safetynet
|
||||||
|
default -> throw new RuntimeException("Feature Extraction is not supported for PDFTron.Element with type: " + element.getType());
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ElementFeatures extractFeaturesWithHash(Element element, String hashObject) throws PDFNetException {
|
||||||
|
return buildImage(element)
|
||||||
|
.hashOfImage(hashObject)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ElementFeatures.Form buildForm(Element element) throws PDFNetException {
|
||||||
|
|
||||||
|
return ElementFeatures.Form.builder()
|
||||||
|
.elementType(element.getType())
|
||||||
|
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
||||||
|
.xObjectType(element.getXObject().getType())
|
||||||
|
.dictOrArrayOrStreamLength(element.getXObject().getType() == 7 ? element.getXObject().getDecodedStream().size() : 0)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ElementFeatures.Image.ImageBuilder<?, ?> buildImage(Element element) throws PDFNetException {
|
||||||
|
|
||||||
|
return ElementFeatures.Image.builder()
|
||||||
|
.elementType(element.getType())
|
||||||
|
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
||||||
|
.dataSize(element.getImageDataSize())
|
||||||
|
.height(element.getImageHeight())
|
||||||
|
.width(element.getImageWidth())
|
||||||
|
.renderingIntent(element.getImageRenderingIntent())
|
||||||
|
.componentNum(element.getComponentNum())
|
||||||
|
.bitsPerComponent(element.getBitsPerComponent());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ElementFeatures.Text buildText(Element element) throws PDFNetException {
|
||||||
|
|
||||||
|
return ElementFeatures.Text.builder()
|
||||||
|
.elementType(element.getType())
|
||||||
|
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
||||||
|
.text(element.getTextString())
|
||||||
|
.font(element.getGState().getFont().getType())
|
||||||
|
.fontsize(element.getGState().getFontSize())
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static ElementFeatures.Path buildPath(Element element) throws PDFNetException {
|
||||||
|
|
||||||
|
return ElementFeatures.Path.builder()
|
||||||
|
.elementType(element.getType())
|
||||||
|
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
||||||
|
.isClippingPath(element.isClippingPath())
|
||||||
|
.isClipWindingFill(element.isClipWindingFill())
|
||||||
|
.isStroked(element.isStroked())
|
||||||
|
.isFilled(element.isFilled())
|
||||||
|
.isWindingFill(element.isWindingFill())
|
||||||
|
.fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), element.getGState().getFillColor()))
|
||||||
|
.strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), element.getGState().getStrokeColor()))
|
||||||
|
.linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), element.getCTM()))
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@ -26,43 +26,6 @@ public class ElementFeatures {
|
|||||||
int elementType;
|
int elementType;
|
||||||
Rectangle2D boundingBox;
|
Rectangle2D boundingBox;
|
||||||
|
|
||||||
public static ElementFeatures extractFeatures(Element element) throws PDFNetException {
|
|
||||||
|
|
||||||
return switch (element.getType()) {
|
|
||||||
case Element.e_path -> Path.builder()
|
|
||||||
.elementType(element.getType())
|
|
||||||
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
|
||||||
.isClippingPath(element.isClippingPath())
|
|
||||||
.isClipWindingFill(element.isClipWindingFill())
|
|
||||||
.isStroked(element.isStroked())
|
|
||||||
.isFilled(element.isFilled())
|
|
||||||
.isWindingFill(element.isWindingFill())
|
|
||||||
.fillColor(Converter.convertColor(element.getGState().getFillColorSpace(), element.getGState().getFillColor()))
|
|
||||||
.strokeColor(Converter.convertColor(element.getGState().getStrokeColorSpace(), element.getGState().getStrokeColor()))
|
|
||||||
.linePath(Converter.convertToGeneralPathAndTransformToInitialUserSpace(element.getPathData(), element.getCTM()))
|
|
||||||
.build();
|
|
||||||
case Element.e_text -> Text.builder()
|
|
||||||
.elementType(element.getType())
|
|
||||||
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
|
||||||
.text(element.getTextString())
|
|
||||||
.font(element.getGState().getFont().getType())
|
|
||||||
.fontsize(element.getGState().getFontSize())
|
|
||||||
.build();
|
|
||||||
case Element.e_image, Element.e_inline_image -> Image.builder()
|
|
||||||
.elementType(element.getType())
|
|
||||||
.boundingBox(Converter.toRectangle2D(element.getBBox()))
|
|
||||||
.dataSize(element.getImageDataSize())
|
|
||||||
.height(element.getImageHeight())
|
|
||||||
.width(element.getImageWidth())
|
|
||||||
.renderingIntent(element.getImageRenderingIntent())
|
|
||||||
.componentNum(element.getComponentNum())
|
|
||||||
.bitsPerComponent(element.getBitsPerComponent())
|
|
||||||
.build();
|
|
||||||
// This technically should never happen, it's a safetynet
|
|
||||||
default -> throw new RuntimeException("Feature Extraction is not supported for PDFTron.Element with type: " + element.getType());
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
public boolean almostMatches(Element element) throws PDFNetException {
|
public boolean almostMatches(Element element) throws PDFNetException {
|
||||||
|
|
||||||
@ -71,6 +34,12 @@ public class ElementFeatures {
|
|||||||
rectsAlmostMatch(element.getBBox());
|
rectsAlmostMatch(element.getBBox());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean almostMatches(ElementFeatures elementFeatures){
|
||||||
|
return elementFeatures.getElementType() == elementType &&
|
||||||
|
elementFeatures.getBoundingBox() != null &&
|
||||||
|
rectsAlmostMatch(elementFeatures.getBoundingBox());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
protected boolean almostEqual(double a, double b) {
|
protected boolean almostEqual(double a, double b) {
|
||||||
|
|
||||||
@ -88,12 +57,22 @@ public class ElementFeatures {
|
|||||||
almostEqual(bBox.getHeight(), boundingBox.getHeight());
|
almostEqual(bBox.getHeight(), boundingBox.getHeight());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private boolean rectsAlmostMatch(Rectangle2D bBox) {
|
||||||
|
// To address the inconsistencies in the calculation of the bounding box we check equality with a tolerance
|
||||||
|
|
||||||
|
return almostEqual(bBox.getX(), boundingBox.getX()) && //
|
||||||
|
almostEqual(bBox.getY(), boundingBox.getY()) && //
|
||||||
|
almostEqual(bBox.getWidth(), boundingBox.getWidth()) && //
|
||||||
|
almostEqual(bBox.getHeight(), boundingBox.getHeight());
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@EqualsAndHashCode(callSuper = true)
|
@EqualsAndHashCode(callSuper = true)
|
||||||
@Getter
|
@Getter
|
||||||
@SuperBuilder
|
@SuperBuilder
|
||||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||||
private static class Text extends ElementFeatures {
|
public static class Text extends ElementFeatures {
|
||||||
|
|
||||||
String text;
|
String text;
|
||||||
int font;
|
int font;
|
||||||
@ -159,7 +138,7 @@ public class ElementFeatures {
|
|||||||
@Getter
|
@Getter
|
||||||
@SuperBuilder
|
@SuperBuilder
|
||||||
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||||
private static class Image extends ElementFeatures {
|
public static class Image extends ElementFeatures {
|
||||||
|
|
||||||
int dataSize;
|
int dataSize;
|
||||||
int height;
|
int height;
|
||||||
@ -167,7 +146,7 @@ public class ElementFeatures {
|
|||||||
int renderingIntent;
|
int renderingIntent;
|
||||||
int componentNum;
|
int componentNum;
|
||||||
int bitsPerComponent;
|
int bitsPerComponent;
|
||||||
|
String hashOfImage;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean almostMatches(Element element) throws PDFNetException {
|
public boolean almostMatches(Element element) throws PDFNetException {
|
||||||
@ -181,6 +160,73 @@ public class ElementFeatures {
|
|||||||
bitsPerComponent == element.getBitsPerComponent();
|
bitsPerComponent == element.getBitsPerComponent();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public boolean almostMatches(ElementFeatures elementFeatures){
|
||||||
|
if(elementFeatures.getClass() != this.getClass()){
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return super.almostMatches(elementFeatures) &&
|
||||||
|
this.dataSize == ((Image) elementFeatures).getDataSize() &&
|
||||||
|
this.height == ((Image) elementFeatures).getHeight() &&
|
||||||
|
this.width == ((Image) elementFeatures).getWidth() &&
|
||||||
|
this.renderingIntent == ((Image) elementFeatures).getRenderingIntent() &&
|
||||||
|
this.componentNum == ((Image) elementFeatures).getComponentNum() &&
|
||||||
|
this.bitsPerComponent == ((Image) elementFeatures).getBitsPerComponent() &&
|
||||||
|
calculateHammingDistance(((Image) elementFeatures).getHashOfImage()) <=4;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper method to calculate the Hamming distance between two hexadecimal strings
|
||||||
|
private int calculateHammingDistance(String hash2) {
|
||||||
|
int distance = 0;
|
||||||
|
int maxLength = Math.max(this.hashOfImage.length(), hash2.length());
|
||||||
|
for (int i = 0; i < maxLength; i++) {
|
||||||
|
char char1 = i < this.hashOfImage.length() ? this.hashOfImage.charAt(i) : '0';
|
||||||
|
char char2 = i < hash2.length() ? hash2.charAt(i) : '0';
|
||||||
|
if (char1 != char2) {
|
||||||
|
distance++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return distance;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@EqualsAndHashCode(callSuper = true)
|
||||||
|
@Getter
|
||||||
|
@SuperBuilder
|
||||||
|
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
|
||||||
|
public static class Form extends ElementFeatures {
|
||||||
|
|
||||||
|
int xObjectType;
|
||||||
|
long dictOrArrayOrStreamLength;
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean almostMatches(Element element) throws PDFNetException {
|
||||||
|
return element.getType() == getElementType() && //
|
||||||
|
element.getBBox() != null && //
|
||||||
|
(super.rectsAlmostMatch(element.getBBox()) || almostRotateMatches(element.getBBox().getRectangle())) &&
|
||||||
|
xObjectType == element.getXObject().getType() &&
|
||||||
|
dictOrArrayOrStreamLength == element.getXObject().getDecodedStream().size();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean almostMatches(ElementFeatures elementFeatures){
|
||||||
|
if(elementFeatures.getClass() != this.getClass()){
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return elementFeatures.getElementType() == getElementType() &&
|
||||||
|
elementFeatures.getBoundingBox() != null &&
|
||||||
|
(super.rectsAlmostMatch(elementFeatures.getBoundingBox()) || almostRotateMatches(elementFeatures.getBoundingBox().getBounds2D())) &&
|
||||||
|
xObjectType == ((Form)elementFeatures).getXObjectType() &&
|
||||||
|
dictOrArrayOrStreamLength == ((Form)elementFeatures).getDictOrArrayOrStreamLength();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private boolean almostRotateMatches(Rectangle2D bBox) {
|
||||||
|
return almostEqual(bBox.getWidth(), getBoundingBox().getHeight()) && //
|
||||||
|
almostEqual(bBox.getHeight(), getBoundingBox().getWidth());
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,96 @@
|
|||||||
|
package com.iqser.red.pdftronlogic.commons;
|
||||||
|
|
||||||
|
import java.awt.Color;
|
||||||
|
import java.awt.image.BufferedImage;
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
|
||||||
|
import javax.imageio.ImageIO;
|
||||||
|
|
||||||
|
import com.pdftron.filters.FilterWriter;
|
||||||
|
import com.pdftron.filters.MemoryFilter;
|
||||||
|
import com.pdftron.pdf.Element;
|
||||||
|
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
public class ImageHashFactory {
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
public String calculate(Element element) {
|
||||||
|
|
||||||
|
com.pdftron.pdf.Image image = new com.pdftron.pdf.Image(element.getXObject());
|
||||||
|
|
||||||
|
byte[] imageBytes = getBytesOfImage(image);
|
||||||
|
ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(imageBytes);
|
||||||
|
BufferedImage image1 = ImageIO.read(byteArrayInputStream);
|
||||||
|
|
||||||
|
String hash = getSimplePHash(image1);
|
||||||
|
|
||||||
|
return hash;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private byte[] getBytesOfImage(com.pdftron.pdf.Image inputImage) {
|
||||||
|
// 0 because the memory filter determines the size
|
||||||
|
var memFilter = new MemoryFilter(0, false);
|
||||||
|
var filterWriter = new FilterWriter(memFilter);
|
||||||
|
|
||||||
|
inputImage.export(filterWriter);
|
||||||
|
filterWriter.flushAll();
|
||||||
|
byte[] res = memFilter.getBuffer();
|
||||||
|
|
||||||
|
memFilter.flushAll();
|
||||||
|
memFilter.destroy();
|
||||||
|
filterWriter.destroy();
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public String getSimplePHash(BufferedImage image) {
|
||||||
|
// Resize the image to a fixed size (e.g., 8x8 pixels)
|
||||||
|
int targetWidth = 8;
|
||||||
|
int targetHeight = 8;
|
||||||
|
BufferedImage resizedImage = new BufferedImage(targetWidth, targetHeight, BufferedImage.TYPE_INT_ARGB);
|
||||||
|
resizedImage.getGraphics().drawImage(image.getScaledInstance(targetWidth, targetHeight, java.awt.Image.SCALE_SMOOTH), 0, 0, targetWidth, targetHeight, null);
|
||||||
|
|
||||||
|
// Convert the image to grayscale
|
||||||
|
BufferedImage grayscaleImage = new BufferedImage(targetWidth, targetHeight, BufferedImage.TYPE_BYTE_GRAY);
|
||||||
|
grayscaleImage.getGraphics().drawImage(resizedImage, 0, 0, null);
|
||||||
|
|
||||||
|
// Calculate the average grayscale pixel value
|
||||||
|
int average = calculateAverage(grayscaleImage);
|
||||||
|
|
||||||
|
// Create a binary hash based on pixel values
|
||||||
|
StringBuilder hashBuilder = new StringBuilder();
|
||||||
|
for (int y = 0; y < targetHeight; y++) {
|
||||||
|
for (int x = 0; x < targetWidth; x++) {
|
||||||
|
int pixelValue = new Color(grayscaleImage.getRGB(x, y)).getRed();
|
||||||
|
if (pixelValue > average) {
|
||||||
|
hashBuilder.append("1");
|
||||||
|
} else {
|
||||||
|
hashBuilder.append("0");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hashBuilder.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// Helper method to calculate the average grayscale pixel value
|
||||||
|
private int calculateAverage(BufferedImage image) {
|
||||||
|
|
||||||
|
int total = 0;
|
||||||
|
int width = image.getWidth();
|
||||||
|
int height = image.getHeight();
|
||||||
|
for (int y = 0; y < height; y++) {
|
||||||
|
for (int x = 0; x < width; x++) {
|
||||||
|
total += new Color(image.getRGB(x, y)).getRed();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return total / (width * height);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -167,7 +167,7 @@ public class InvisibleElementRemovalService {
|
|||||||
boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
|
boolean inClippingPath = context.clippingPathStack().almostIntersects(rect.getX1(), rect.getY1(), rect.getWidth(), rect.getHeight());
|
||||||
|
|
||||||
if (!context.delta() && inClippingPath) {
|
if (!context.delta() && inClippingPath) {
|
||||||
context.visibleElements().add(ElementFeatures.extractFeatures(imageElement));
|
context.visibleElements().add(ElementFeatureFactory.extractFeatures(imageElement));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (context.delta() ^ inClippingPath) {
|
if (context.delta() ^ inClippingPath) {
|
||||||
@ -192,7 +192,7 @@ public class InvisibleElementRemovalService {
|
|||||||
boolean isTextVisible = isTextRenderedVisibly(gState, textBBox, context);
|
boolean isTextVisible = isTextRenderedVisibly(gState, textBBox, context);
|
||||||
|
|
||||||
if (inClippingPath && isTextVisible) {
|
if (inClippingPath && isTextVisible) {
|
||||||
context.visibleElements().add(ElementFeatures.extractFeatures(textElement));
|
context.visibleElements().add(ElementFeatureFactory.extractFeatures(textElement));
|
||||||
}
|
}
|
||||||
if (!context.delta()) {
|
if (!context.delta()) {
|
||||||
if (inClippingPath && isTextVisible) {
|
if (inClippingPath && isTextVisible) {
|
||||||
@ -291,7 +291,7 @@ public class InvisibleElementRemovalService {
|
|||||||
context.overlappedElements().addAll(currentOverlappedElements);
|
context.overlappedElements().addAll(currentOverlappedElements);
|
||||||
context.visibleElements().removeAll(currentOverlappedElements);
|
context.visibleElements().removeAll(currentOverlappedElements);
|
||||||
}
|
}
|
||||||
context.visibleElements().add(ElementFeatures.extractFeatures(pathElement));
|
context.visibleElements().add(ElementFeatureFactory.extractFeatures(pathElement));
|
||||||
if (!context.delta()) {
|
if (!context.delta()) {
|
||||||
writer.writeElement(pathElement);
|
writer.writeElement(pathElement);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,289 @@
|
|||||||
|
package com.iqser.red.pdftronlogic.commons;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.TreeSet;
|
||||||
|
|
||||||
|
import com.pdftron.common.PDFNetException;
|
||||||
|
import com.pdftron.pdf.Element;
|
||||||
|
import com.pdftron.pdf.ElementReader;
|
||||||
|
import com.pdftron.pdf.ElementWriter;
|
||||||
|
import com.pdftron.pdf.PDFDoc;
|
||||||
|
import com.pdftron.pdf.Page;
|
||||||
|
import com.pdftron.pdf.PageIterator;
|
||||||
|
import com.pdftron.sdf.SDFDoc;
|
||||||
|
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
import lombok.experimental.UtilityClass;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@UtilityClass
|
||||||
|
@Slf4j
|
||||||
|
public class WatermarkRemovalService {
|
||||||
|
|
||||||
|
final static double AREA_THRESHOLD = 0.6; // multiplied with page area
|
||||||
|
final static double OCCURING_ON_PAGES_THRESHOLD_FACTOR = 0.75; // multiplied with number of pages
|
||||||
|
|
||||||
|
final static int MIN_PAGES_THRESHOLD = 3;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The method remove watermark works only for Documents with size greater than MIN_PAGES_THRESHOLD.
|
||||||
|
* First the possible watermarks (big XObjects or Images) will be detected and then checked if those appear on most pages according to the
|
||||||
|
* OCCURING_ON_PAGES_THRESHOLD_FACTOR by using image hashing for similarity and size and stream size of the xobjects.
|
||||||
|
* If so, these detected and confirmed will not be written to the pdf file.
|
||||||
|
* @param pdfFile PDFFile to remove watermarks
|
||||||
|
* @param out The OutputStream the final file will be written to
|
||||||
|
*/
|
||||||
|
@SneakyThrows
|
||||||
|
public void removeWatermarks(InputStream pdfFile, OutputStream out) {
|
||||||
|
|
||||||
|
PDFDoc pdfDoc = new PDFDoc(pdfFile);
|
||||||
|
|
||||||
|
if(pdfDoc.getPageCount() < MIN_PAGES_THRESHOLD){
|
||||||
|
log.info("Document page count {} is below threshold {}", pdfDoc.getPageCount(), MIN_PAGES_THRESHOLD);
|
||||||
|
} else {
|
||||||
|
Map<Long, List<ElementFeatures>> formObjectsForPages = findAllFormObjectsAndImages(pdfDoc);
|
||||||
|
|
||||||
|
List<ElementFeatures> watermarkElementFeatures = filterSameFormObjectsOccuringOnMostPages(formObjectsForPages);
|
||||||
|
|
||||||
|
if (watermarkElementFeatures.size() > 0) {
|
||||||
|
log.info("Watermark found and will be removed!");
|
||||||
|
removeAllWatermarks(pdfDoc, watermarkElementFeatures);
|
||||||
|
} else {
|
||||||
|
log.info("No watermark found!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
pdfDoc.save(out, SDFDoc.SaveMode.LINEARIZED, null);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
} finally {
|
||||||
|
pdfDoc.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private Map<Long, List<ElementFeatures>> findAllFormObjectsAndImages(PDFDoc pdfDoc) {
|
||||||
|
|
||||||
|
List<ElementFeatures> formObjectsOccuringMoreThanOnceOnAPage = new LinkedList<>();
|
||||||
|
Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>();
|
||||||
|
Set<Long> visitedXObjIds = new TreeSet<>();
|
||||||
|
|
||||||
|
ElementReader reader = new ElementReader();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
|
||||||
|
|
||||||
|
Page page = iterator.next();
|
||||||
|
|
||||||
|
double minAreaCoveringFromPage = AREA_THRESHOLD * page.getPageHeight() * page.getPageWidth();
|
||||||
|
|
||||||
|
LinkedList<ElementFeatures> elementFeaturesLinkedList = new LinkedList<>();
|
||||||
|
|
||||||
|
reader.begin(page);
|
||||||
|
for (Element element = reader.next(); element != null; element = reader.next()) {
|
||||||
|
processElement(element, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringFromPage);
|
||||||
|
}
|
||||||
|
|
||||||
|
formObjectsAndImagesForPages.put(page.getSDFObj().getObjNum(), elementFeaturesLinkedList);
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.destroy();
|
||||||
|
|
||||||
|
return formObjectsAndImagesForPages;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void processElement(Element element,
|
||||||
|
Set<Long> visitedXObjIds,
|
||||||
|
List<ElementFeatures> elementFeaturesLinkedList,
|
||||||
|
List<ElementFeatures> formObjectsOccuringMoreThanOnceOnAPage,
|
||||||
|
double minAreaCoveringPage) throws PDFNetException {
|
||||||
|
|
||||||
|
if (element.getBBox() == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringPage) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (element.getType() == Element.e_form) {
|
||||||
|
processXObject(element, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage);
|
||||||
|
} else if (element.getType() == Element.e_image || element.getType() == Element.e_inline_image) {
|
||||||
|
processImages(element, elementFeaturesLinkedList);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void processImages(Element element, List<ElementFeatures> elementFeaturesLinkedList) {
|
||||||
|
|
||||||
|
String hashOfImage = ImageHashFactory.calculate(element);
|
||||||
|
ElementFeatures elementFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashOfImage);
|
||||||
|
elementFeaturesLinkedList.add(elementFeatures);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void processXObject(Element element,
|
||||||
|
Set<Long> visitedXObjIds,
|
||||||
|
List<ElementFeatures> elementFeaturesLinkedList,
|
||||||
|
List<ElementFeatures> formObjectsOccuringMoreThanOnceOnAPage,
|
||||||
|
double minAreaCoveringPage) {
|
||||||
|
|
||||||
|
if (visitedXObjIds.add(element.getXObject().getObjNum())) {
|
||||||
|
ElementReader xObjectReader = new ElementReader();
|
||||||
|
xObjectReader.begin(element.getXObject());
|
||||||
|
for (Element element1 = xObjectReader.next(); element1 != null; element1 = xObjectReader.next()) {
|
||||||
|
processElement(element1, visitedXObjIds, elementFeaturesLinkedList, formObjectsOccuringMoreThanOnceOnAPage, minAreaCoveringPage);
|
||||||
|
}
|
||||||
|
elementFeaturesLinkedList.add(ElementFeatureFactory.extractFeatures(element));
|
||||||
|
xObjectReader.destroy();
|
||||||
|
} else {
|
||||||
|
elementFeaturesLinkedList.add(ElementFeatureFactory.extractFeatures(element));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
parameter
|
||||||
|
*/
|
||||||
|
private List<ElementFeatures> filterSameFormObjectsOccuringOnMostPages(Map<Long, List<ElementFeatures>> formObjectsPerPage) {
|
||||||
|
|
||||||
|
int pageCount = formObjectsPerPage.keySet().size();
|
||||||
|
int minPagesFilter = (int) (OCCURING_ON_PAGES_THRESHOLD_FACTOR * pageCount);
|
||||||
|
|
||||||
|
return formObjectsPerPage.values()
|
||||||
|
.stream()
|
||||||
|
.flatMap(Collection::stream)
|
||||||
|
.filter(elementFeature -> formObjectsPerPage.values()
|
||||||
|
.stream()
|
||||||
|
.filter(elementFeaturesOnPage -> elementFeaturesOnPage.stream().anyMatch(elementFeature::almostMatches))
|
||||||
|
.count() >= minPagesFilter)
|
||||||
|
.toList();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) {
|
||||||
|
|
||||||
|
ElementReader reader = new ElementReader();
|
||||||
|
ElementWriter writer = new ElementWriter();
|
||||||
|
Set<Long> visitedXObjIds = new TreeSet<>();
|
||||||
|
|
||||||
|
for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
|
||||||
|
|
||||||
|
Page page = iterator.next();
|
||||||
|
|
||||||
|
writeAllElementsExceptWatermarks(page, reader, writer, watermarksElementFeaturesList, visitedXObjIds);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.destroy();
|
||||||
|
writer.destroy();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void writeAllElementsExceptWatermarks(Page page,
|
||||||
|
ElementReader reader,
|
||||||
|
ElementWriter writer,
|
||||||
|
List<ElementFeatures> watermarksElementFeaturesList,
|
||||||
|
Set<Long> visitedXObjIds) {
|
||||||
|
|
||||||
|
reader.begin(page);
|
||||||
|
writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
|
||||||
|
processElements(page, reader, writer, watermarksElementFeaturesList, visitedXObjIds);
|
||||||
|
writer.end();
|
||||||
|
reader.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void processElements(Page page,
|
||||||
|
ElementReader reader,
|
||||||
|
ElementWriter writer,
|
||||||
|
List<ElementFeatures> watermarksElementFeaturesList,
|
||||||
|
Set<Long> visitedXObjIds) throws PDFNetException {
|
||||||
|
|
||||||
|
double minAreaCoveringFromPage = AREA_THRESHOLD * page.getPageHeight() * page.getPageWidth();
|
||||||
|
for (Element element = reader.next(); element != null; element = reader.next()) {
|
||||||
|
|
||||||
|
switch (element.getType()) {
|
||||||
|
case Element.e_image, Element.e_inline_image -> {
|
||||||
|
if (element.getBBox() == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (element.getBBox().getHeight() * element.getBBox().getWidth() < minAreaCoveringFromPage) {
|
||||||
|
writer.writeElement(element);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
removeImages(element, writer, watermarksElementFeaturesList);
|
||||||
|
}
|
||||||
|
case Element.e_form -> processForms(page, element, reader, writer, watermarksElementFeaturesList, visitedXObjIds);
|
||||||
|
default -> writer.writeElement(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
private void removeImages(Element element, ElementWriter writer, List<ElementFeatures> watermarksElementFeaturesList) {
|
||||||
|
|
||||||
|
String hashValueOfImage = ImageHashFactory.calculate(element);
|
||||||
|
ElementFeatures imageFeatures = ElementFeatureFactory.extractFeaturesWithHash(element, hashValueOfImage);
|
||||||
|
for (ElementFeatures elementFeatures : watermarksElementFeaturesList) {
|
||||||
|
if (elementFeatures.almostMatches(imageFeatures)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.writeElement(element);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private void processForms(Page page,
|
||||||
|
Element element,
|
||||||
|
ElementReader reader,
|
||||||
|
ElementWriter writer,
|
||||||
|
List<ElementFeatures> watermarksElementFeaturesList,
|
||||||
|
Set<Long> visitedXObjIds) throws PDFNetException {
|
||||||
|
|
||||||
|
for (ElementFeatures elementFeatures : watermarksElementFeaturesList) {
|
||||||
|
if (elementFeatures.almostMatches(element)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.writeElement(element);
|
||||||
|
|
||||||
|
if (!visitedXObjIds.contains(element.getXObject().getObjNum())) {
|
||||||
|
visitedXObjIds.add(element.getXObject().getObjNum());
|
||||||
|
// writer needs to be newly initialized when entering a new content stream
|
||||||
|
// see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
|
||||||
|
ElementWriter formWriter = new ElementWriter();
|
||||||
|
reader.formBegin();
|
||||||
|
formWriter.begin(element.getXObject());
|
||||||
|
|
||||||
|
reader.clearChangeList();
|
||||||
|
formWriter.setDefaultGState(reader);
|
||||||
|
|
||||||
|
processElements(page, reader, formWriter, watermarksElementFeaturesList, visitedXObjIds);
|
||||||
|
formWriter.end();
|
||||||
|
formWriter.destroy();
|
||||||
|
reader.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,62 @@
|
|||||||
|
package com.iqser.red.pdftronlogic.commons;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.Locale;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.platform.commons.util.StringUtils;
|
||||||
|
|
||||||
|
import com.pdftron.pdf.PDFNet;
|
||||||
|
|
||||||
|
import lombok.SneakyThrows;
|
||||||
|
|
||||||
|
class WatermarkRemovalServiceTest {
|
||||||
|
|
||||||
|
@SneakyThrows
|
||||||
|
@Test
|
||||||
|
void removeWatermarks() {
|
||||||
|
|
||||||
|
PDFNet.addResourceSearchPath("C:/Users/RaphaelArnold/knecon/pdftron/ocrirismodule/Lib");
|
||||||
|
PDFNet.initialize("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a");
|
||||||
|
|
||||||
|
String filename = "files/PO_82_277_H_2013 (1).pdf";
|
||||||
|
|
||||||
|
String tmpFilename = createTmpFileName(filename, "WATERMARK_REMOVAL");
|
||||||
|
try (var in = this.getClass().getClassLoader().getResourceAsStream(filename); var out = new FileOutputStream(tmpFilename)) {
|
||||||
|
|
||||||
|
{
|
||||||
|
System.out.println(tmpFilename);
|
||||||
|
WatermarkRemovalService.removeWatermarks(in, out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private static boolean isWindows() {
|
||||||
|
|
||||||
|
return System.getProperty("os.name").toLowerCase(Locale.ROOT).contains("windows");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static String getTemporaryDirectory() {
|
||||||
|
|
||||||
|
String tmpdir = System.getProperty("java.io.tmpdir");
|
||||||
|
if (isWindows() && StringUtils.isNotBlank(tmpdir)) {
|
||||||
|
return tmpdir;
|
||||||
|
}
|
||||||
|
return "/tmp";
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static String createTmpFileName(String filename, String suffix) {
|
||||||
|
|
||||||
|
return Path.of(getTemporaryDirectory()).resolve(Path.of(filename).getFileName()).toString().replace(".pdf", "_" + suffix + ".pdf");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
BIN
src/test/resources/files/PO_82_277_H_2013 (1).pdf
Normal file
BIN
src/test/resources/files/PO_82_277_H_2013 (1).pdf
Normal file
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user