changes in export_example_pages.py as well as removing unused imports in table_parsing.py

cv_analysis/table_parsing.py

@@ -1,16 +1,19 @@
+
 from functools import partial
 from itertools import chain, starmap
 from operator import attrgetter
 
-from os.path import join
+
 import cv2
 import numpy as np
+
 from pdf2image import pdf2image
 
+
 from cv_analysis.utils.display import show_mpl
 from cv_analysis.utils.draw import draw_rectangles
 from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d
-from cv_analysis.utils.deskew import deskew_histbased
+from cv_analysis.utils.deskew import deskew_histbased, deskew
 from cv_analysis.utils.filters import is_large_enough
 from cv_analysis.utils.visual_logging import vizlogger
 from cv_analysis.layout_parsing import parse_layout
@@ -26,12 +29,12 @@ def add_external_contours(image, img):
 
 
 def extend_lines():
-    #TODO
+    # TODO
     pass
 
 
 def make_table_block_mask():
-    #TODO
+    # TODO
     pass
 
 
@@ -164,8 +167,8 @@ def parse_table(image: np.array, show=False):
     table_layout_boxes = find_table_layout_boxes(image)
 
     image = isolate_vertical_and_horizontal_components(image)
-    #image = add_external_contours(image, image)
+    # image = add_external_contours(image, image)
-    #vizlogger.debug(image, "external_contours_added.png")
+    # vizlogger.debug(image, "external_contours_added.png")
 
     _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S)
 
@@ -198,3 +201,5 @@ def tables_in_image(cropped_image):
         return True, table_rects
     else:
         return False, None
+
+

cv_analysis/test/scripts/export_example_pages.py

@@ -1,3 +1,4 @@
+import hashlib
 import os
 from os import path
 import pandas as pd
@@ -5,7 +6,7 @@ from pdf2image import convert_from_path
 from itertools import chain
 import json
 from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS_FOR_TESTING
-from cv_analysis.utils.deduplicate_pdfs import hash_pdf_files
+
 
 def read_json(path):
     with open(path, encoding='utf-8') as file:
@@ -13,23 +14,6 @@ def read_json(path):
     return data
 
 
-# def collect_metadata(example_pages, save=False):
-#     metadata = []
-#     i = 0
-#     for name, document_sections in example_pages.items():
-#         for pages in document_sections:
-#             span = list(range(pages[0], pages[1] + 1))
-#             for page_nr in span:
-#                 metadata.append(["fig_table" + str(i), name, page_nr])
-#                 i += 1
-#     if save:
-#         df = pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"])
-#         df.to_csv("/exported_files/test_pages.csv")
-#     else:
-#         return pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"])
-
-
-
 def collect_metadata(example_pages, save=False):
     metadata = []
     make_metadata_entry = make_metadata_entry_maker()
@@ -73,8 +57,21 @@ def split_pdf(example_pages):
                 image.save(fp=fp, dpi=(300, 300))
                 i += 1
 
-def rename_files_with_hash(example_pages, hashes):
 
+def find_hash(file_path):
+    BLOCK_SIZE = 65536
+
+    file_hash = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        fb = f.read(BLOCK_SIZE)
+        while len(fb) > 0:
+            file_hash.update(fb)
+            fb = f.read(BLOCK_SIZE)
+
+    return file_hash.hexdigest()
+
+
+def rename_files_with_hash(example_pages):
     files_to_rename = list(example_pages.keys())
     folder = HASHED_PDFS_FOR_TESTING
 
@@ -88,9 +85,9 @@ def rename_files_with_hash(example_pages, hashes):
             only_name = path.splitext(file)[0]
 
             # Adding the new name with extension
-            new_base = only_name + '_new' + '.txt'
+            hash = find_hash(old_name)
             # construct full file path
-            new_name = path.join(folder, new_base)
+            new_name = path.join(folder, hash + ".pdf")
 
             # Renaming the file
             os.rename(old_name, new_name)
@@ -99,17 +96,12 @@ def rename_files_with_hash(example_pages, hashes):
     res = os.listdir(folder)
     print(res)
 
-def hash_pdfs(example_pages):
-    pdf_paths = list(path.join(PDF_FOR_TESTING, pdf_name) for pdf_name in example_pages.keys())
-    hashes = hash_pdf_files(paths=pdf_paths, verbose=0)
-    example_pages = dict(zip(hashes, example_pages.values()))
-    return example_pages
 
 def main():
     examples_pages = read_json(path.join(TEST_DATA_DIR, "example_pages.json"))
-    # examples_pages = hash_pdfs(examples_pages)
+    rename_files_with_hash(examples_pages)
-    collect_metadata(examples_pages, save=True)
+    #collect_metadata(examples_pages, save=True)
-    split_pdf(examples_pages)
+    #split_pdf(examples_pages)
 
 
 if __name__ == "__main__":