RED-9746: Document hardly editable

* revert quadtree lookup, since the lib does not seem to work reliably, also, no significant speed boost
* check each individual glyph instead of only a text run and remember past overlaps in glyph
* added logic to extract all glyphs exactly
* check for optional content or transparency in form objects and marked content

src/main/java/com/iqser/red/pdftronlogic/commons/InvisibleElementRemovalService.java

@@ -154,10 +154,10 @@ public class InvisibleElementRemovalService {
     private void execute(PDFDoc pdfDoc, boolean delta, boolean removePaths, Set<String> markedContentToIgnore) {
 
         log.info("Start removing invisible Elements");
-        try (ElementWriter writer = new ElementWriter(); ElementReader reader = new ElementReader()) {
+        try (PageIterator iterator = pdfDoc.getPageIterator(); ElementWriter writer = new ElementWriter(); ElementReader reader = new ElementReader()) {
             Set<Long> visitedXObjIds = new TreeSet<>();
 
-            for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
+            while (iterator.hasNext()) {
 
                 Page page = iterator.next();
 
@@ -182,6 +182,7 @@ public class InvisibleElementRemovalService {
 
                 removeOverlappedElements(page, writer, context);
             }
+
         }
         log.info("Finished removing invisible Elements");
     }
@@ -250,9 +251,8 @@ public class InvisibleElementRemovalService {
                 if (!(context.markedContentStack.contextHasTransparency()
                       || imageFeatures.isTransparent()
                       || imageFeatures.isImageMask()
-                      || imageFeatures.isSoftMask()
+                      || imageFeatures.isSoftMask())) {
-                      || imageFeatures.isMasked())) {
+                    calculateOverlaps(context, imageFeatures, imageFeatures.isMasked());
-                    calculateOverlaps(context, imageFeatures);
                 }
                 context.visibleElements().add(imageFeatures);
             }
@@ -387,7 +387,7 @@ public class InvisibleElementRemovalService {
 
                 if (inClippingPath) {
                     if (!context.markedContentStack.contextHasTransparency() && isFilledAndNonTransparent(pathElement)) {
-                        calculateOverlaps(context, pathFeatures);
+                        calculateOverlaps(context, pathFeatures, false);
                     }
                     context.visibleElements().add(ElementFeatureFactory.extractFeatures(pathElement));
                 }
@@ -410,9 +410,9 @@ public class InvisibleElementRemovalService {
     }
 
 
-    private void calculateOverlaps(InvisibleElementRemovalContext context, ElementFeatures elementFeatures) {
+    private void calculateOverlaps(InvisibleElementRemovalContext context, ElementFeatures elementFeatures, boolean textOnly) {
 
-        List<ElementFeatures> currentOverlappedElements = context.visibleElements().findOverlapped(elementFeatures);
+        List<ElementFeatures> currentOverlappedElements = context.visibleElements().findOverlapped(elementFeatures, textOnly);
         context.overlappedElements().addAll(currentOverlappedElements);
         context.visibleElements().removeAll(currentOverlappedElements);
     }

src/main/java/com/iqser/red/pdftronlogic/commons/OCGWatermarkRemovalService.java

@@ -5,6 +5,7 @@ import com.pdftron.pdf.*;
 import com.pdftron.pdf.ocg.Group;
 import com.pdftron.pdf.ocg.OCMD;
 import com.pdftron.sdf.Obj;
+
 import lombok.SneakyThrows;
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
@@ -27,8 +28,9 @@ public class OCGWatermarkRemovalService {
 
     @SneakyThrows
     private boolean hasOCGWatermarks(PDFDoc pdfDoc) {
+
         Obj ocgs = pdfDoc.getOCGs();
-        if(ocgs != null) {
+        if (ocgs != null) {
             for (int i = 0; i < ocgs.size(); i++) {
                 Group group = new Group(ocgs.getAt(i));
                 if (group.isValid() && group.getName().equals("Watermark")) {
@@ -43,24 +45,21 @@ public class OCGWatermarkRemovalService {
     @SneakyThrows
     private void removeOCGWatermarks(PDFDoc pdfDoc) {
 
-        try(ElementReader reader = new ElementReader();
+        try (PageIterator iterator = pdfDoc.getPageIterator(); ElementReader reader = new ElementReader(); ElementWriter writer = new ElementWriter()) {
-            ElementWriter writer = new ElementWriter()) {
             Set<Long> visitedXObjIds = new TreeSet<>();
 
-            for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
+            while (iterator.hasNext()) {
 
                 Page page = iterator.next();
                 writeAllElementsExceptWatermarks(page, reader, writer, visitedXObjIds);
             }
+
         }
     }
 
 
     @SneakyThrows
-    private void writeAllElementsExceptWatermarks(Page page,
+    private void writeAllElementsExceptWatermarks(Page page, ElementReader reader, ElementWriter writer, Set<Long> visitedXObjIds) {
-                                                  ElementReader reader,
-                                                  ElementWriter writer,
-                                                  Set<Long> visitedXObjIds) {
 
         reader.begin(page);
         writer.begin(page, ElementWriter.e_replacement, false, true, page.getResourceDict());
@@ -70,10 +69,7 @@ public class OCGWatermarkRemovalService {
     }
 
 
-    private void processElements(Page page,
+    private void processElements(Page page, ElementReader reader, ElementWriter writer, Set<Long> visitedXObjIds) throws PDFNetException {
-                                 ElementReader reader,
-                                 ElementWriter writer,
-                                 Set<Long> visitedXObjIds) throws PDFNetException {
 
         for (Element element = reader.next(); element != null; element = reader.next()) {
 
@@ -91,6 +87,7 @@ public class OCGWatermarkRemovalService {
 
     @SneakyThrows
     private boolean inOCGWatermark(Element element) {
+
         var xObj = element.getXObject();
         if (xObj != null) {
             Obj oc = xObj.findObj("OC");
@@ -109,12 +106,7 @@ public class OCGWatermarkRemovalService {
 
 
     @SneakyThrows
-    private void processForms(Page page,
+    private void processForms(Page page, Element element, ElementReader reader, ElementWriter writer, Set<Long> visitedXObjIds) {
-                              Element element,
-                              ElementReader reader,
-                              ElementWriter writer,
-                              Set<Long> visitedXObjIds) {
-
 
         writer.writeElement(element);
 
@@ -122,7 +114,7 @@ public class OCGWatermarkRemovalService {
             visitedXObjIds.add(element.getXObject().getObjNum());
             // writer needs to be newly initialized when entering a new content stream
             // see ElementEditTest in PDFTron (https://www.pdftron.com/documentation/samples/android/java/ElementEditTest)
-            try(ElementWriter formWriter = new ElementWriter()) {
+            try (ElementWriter formWriter = new ElementWriter()) {
                 reader.formBegin();
                 formWriter.begin(element.getXObject());
 

src/main/java/com/iqser/red/pdftronlogic/commons/PdfTextExtraction.java

@@ -26,10 +26,9 @@ public class PdfTextExtraction {
 
     private static String execute(PDFDoc pdfDoc) throws PDFNetException {
 
-        try (TextExtractor extractor = new TextExtractor()) {
+        try (PageIterator iterator = pdfDoc.getPageIterator(); TextExtractor extractor = new TextExtractor()) {
             List<String> texts = new ArrayList<>();
 
-            PageIterator iterator = pdfDoc.getPageIterator();
             while (iterator.hasNext()) {
                 Page page = iterator.next();
                 extractor.begin(page);

src/main/java/com/iqser/red/pdftronlogic/commons/WatermarkRemovalService.java

@@ -95,9 +95,8 @@ public class WatermarkRemovalService {
         Map<Long, List<ElementFeatures>> formObjectsAndImagesForPages = new HashMap<>();
         Set<Long> visitedXObjIds = new TreeSet<>();
 
-        try (ElementReader reader = new ElementReader()) {
+        try (ElementReader reader = new ElementReader(); PageIterator iterator = pdfDoc.getPageIterator()) {
-
+            while (iterator.hasNext()) {
-            for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
 
                 Page page = iterator.next();
 
@@ -270,10 +269,10 @@ public class WatermarkRemovalService {
     @SneakyThrows
     private void removeAllWatermarks(PDFDoc pdfDoc, List<ElementFeatures> watermarksElementFeaturesList) {
 
-        try (ElementReader reader = new ElementReader(); ElementWriter writer = new ElementWriter()) {
+        try (PageIterator iterator = pdfDoc.getPageIterator(); ElementReader reader = new ElementReader(); ElementWriter writer = new ElementWriter()) {
             Set<Long> visitedXObjIds = new TreeSet<>();
 
-            for (PageIterator iterator = pdfDoc.getPageIterator(); iterator.hasNext(); ) {
+            while (iterator.hasNext()) {
 
                 Page page = iterator.next();
 

src/main/java/com/iqser/red/pdftronlogic/commons/features/ElementFeatureFactory.java

@@ -158,33 +158,34 @@ public class ElementFeatureFactory {
 
         List<GlyphInfo> glyphs = new ArrayList<>();
 
-        CharIterator charIterator = textElement.getCharIterator();
+        try (CharIterator charIterator = textElement.getCharIterator()) {
-        while (charIterator.hasNext()) {
+            while (charIterator.hasNext()) {
-            CharData charData = charIterator.next();
+                CharData charData = charIterator.next();
-            long charCode = charData.getCharCode();
+                long charCode = charData.getCharCode();
-            String glyphText = new String(font.mapToUnicode(charCode));
+                String glyphText = new String(font.mapToUnicode(charCode));
 
-            if (Character.isWhitespace(glyphText.charAt(0))) {
+                if (Character.isWhitespace(glyphText.charAt(0))) {
-                continue;
-            }
-
-            try (Matrix2D fontMatrix = computeFontMatrix(charData, textElement, font); //
-                 Matrix2D glyphMatrix = textElement.getCTM()//
-                         .multiply(textElement.getTextMatrix())//
-                         .multiply(fontMatrix)) {
-                PathData pathData = font.getGlyphPath(charCode, true, glyphMatrix);
-                if (pathData.getOperators().length == 1 && pathData.getOperators()[0] == 6) {
-                    // This happens for some chinese characters or whitespaces, don't know why...
                     continue;
                 }
-                GeneralPath glyphPath = Converter.convertToGeneralPath(pathData);
-                GlyphInfo.GlyphInfoBuilder glyphInfo = GlyphInfo.builder().unicode(glyphText).bbox(glyphPath.getBounds2D());
 
-                if (includePathData) {
+                try (Matrix2D fontMatrix = computeFontMatrix(charData, textElement, font); //
-                    glyphInfo.pathData(pathData);
+                     Matrix2D glyphMatrix = textElement.getCTM()//
+                             .multiply(textElement.getTextMatrix())//
+                             .multiply(fontMatrix)) {
+                    PathData pathData = font.getGlyphPath(charCode, true, glyphMatrix);
+                    if (pathData.getOperators().length == 1 && pathData.getOperators()[0] == 6) {
+                        // This happens for some chinese characters or whitespaces, don't know why...
+                        continue;
+                    }
+                    GeneralPath glyphPath = Converter.convertToGeneralPath(pathData);
+                    GlyphInfo.GlyphInfoBuilder glyphInfo = GlyphInfo.builder().unicode(glyphText).bbox(glyphPath.getBounds2D());
+
+                    if (includePathData) {
+                        glyphInfo.pathData(pathData);
+                    }
+
+                    glyphs.add(glyphInfo.build());
                 }
-
-                glyphs.add(glyphInfo.build());
             }
         }
 

src/main/java/com/iqser/red/pdftronlogic/commons/lookup/ElementFeatureLookup.java

@@ -67,9 +67,9 @@ public class ElementFeatureLookup {
     }
 
 
-    public List<ElementFeatures> findOverlapped(ElementFeatures overlappingElement) {
+    public List<ElementFeatures> findOverlapped(ElementFeatures overlappingElement, boolean textOnly) {
 
-        OverlapVisitor overlapVisitor = new OverlapVisitor(overlappingElement);
+        OverlapVisitor overlapVisitor = new OverlapVisitor(overlappingElement, textOnly);
         forEach(overlapVisitor::visitItem);
         return overlapVisitor.getOverlappedElementFeatures();
     }

src/main/java/com/iqser/red/pdftronlogic/commons/lookup/OverlapVisitor.java

@@ -5,6 +5,7 @@ import java.util.List;
 
 import com.iqser.red.pdftronlogic.commons.ComparisonUtils;
 import com.iqser.red.pdftronlogic.commons.features.ElementFeatures;
+import com.pdftron.pdf.Element;
 
 import lombok.AccessLevel;
 import lombok.Getter;
@@ -17,6 +18,8 @@ public class OverlapVisitor implements ElementFeatureVisitor {
 
     ElementFeatures overlappingElement;
 
+    boolean textOnly;
+
     @Getter
     List<ElementFeatures> overlappedElementFeatures = new LinkedList<>();
 
@@ -24,6 +27,10 @@ public class OverlapVisitor implements ElementFeatureVisitor {
     @Override
     public void visitItem(ElementFeatures features) {
 
+        if (textOnly && features.getElementType() != Element.e_text) {
+            return;
+        }
+
         if (ComparisonUtils.padRectangle(features.getBoundingBox()).intersects(ComparisonUtils.padRectangle(overlappingElement.getBoundingBox()))) {
             if (features.testOverlapped(overlappingElement)) {
                 overlappedElementFeatures.add(features);

src/test/java/com/iqser/red/pdftronlogic/commons/VisualEqualityTest.java

@@ -6,7 +6,6 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
@@ -24,11 +23,8 @@ import com.iqser.red.pdftronlogic.commons.rendering.ImageFile;
 import com.pdftron.pdf.PDFDoc;
 import com.pdftron.pdf.PDFNet;
 import com.pdftron.sdf.SDFDoc;
-import com.sun.jna.Memory;
-import com.sun.jna.Native;
 import com.sun.jna.NativeLibrary;
 import com.sun.jna.Pointer;
-import com.sun.jna.ptr.PointerByReference;
 
 import lombok.SneakyThrows;
 import net.sourceforge.lept4j.Box;