From f5a75f39498d4dfbd026b7779b33006e851d048c Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 24 May 2022 16:20:52 +0200 Subject: [PATCH] changes in export_example_pages.py as well as removing unused imports in table_parsing.py --- cv_analysis/table_parsing.py | 17 ++++--- .../test/scripts/export_example_pages.py | 50 ++++++++----------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index dbb8334..2b6344b 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -1,16 +1,19 @@ + from functools import partial from itertools import chain, starmap from operator import attrgetter -from os.path import join + import cv2 import numpy as np + from pdf2image import pdf2image + from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d -from cv_analysis.utils.deskew import deskew_histbased +from cv_analysis.utils.deskew import deskew_histbased, deskew from cv_analysis.utils.filters import is_large_enough from cv_analysis.utils.visual_logging import vizlogger from cv_analysis.layout_parsing import parse_layout @@ -26,12 +29,12 @@ def add_external_contours(image, img): def extend_lines(): - #TODO + # TODO pass def make_table_block_mask(): - #TODO + # TODO pass @@ -164,8 +167,8 @@ def parse_table(image: np.array, show=False): table_layout_boxes = find_table_layout_boxes(image) image = isolate_vertical_and_horizontal_components(image) - #image = add_external_contours(image, image) - #vizlogger.debug(image, "external_contours_added.png") + # image = add_external_contours(image, image) + # vizlogger.debug(image, "external_contours_added.png") _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) @@ -198,3 +201,5 @@ def tables_in_image(cropped_image): return True, table_rects else: return False, None + + diff --git a/cv_analysis/test/scripts/export_example_pages.py b/cv_analysis/test/scripts/export_example_pages.py index 65ffbb0..79dcd04 100644 --- a/cv_analysis/test/scripts/export_example_pages.py +++ b/cv_analysis/test/scripts/export_example_pages.py @@ -1,3 +1,4 @@ +import hashlib import os from os import path import pandas as pd @@ -5,7 +6,7 @@ from pdf2image import convert_from_path from itertools import chain import json from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS_FOR_TESTING -from cv_analysis.utils.deduplicate_pdfs import hash_pdf_files + def read_json(path): with open(path, encoding='utf-8') as file: @@ -13,23 +14,6 @@ def read_json(path): return data -# def collect_metadata(example_pages, save=False): -# metadata = [] -# i = 0 -# for name, document_sections in example_pages.items(): -# for pages in document_sections: -# span = list(range(pages[0], pages[1] + 1)) -# for page_nr in span: -# metadata.append(["fig_table" + str(i), name, page_nr]) -# i += 1 -# if save: -# df = pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) -# df.to_csv("/exported_files/test_pages.csv") -# else: -# return pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) - - - def collect_metadata(example_pages, save=False): metadata = [] make_metadata_entry = make_metadata_entry_maker() @@ -73,8 +57,21 @@ def split_pdf(example_pages): image.save(fp=fp, dpi=(300, 300)) i += 1 -def rename_files_with_hash(example_pages, hashes): +def find_hash(file_path): + BLOCK_SIZE = 65536 + + file_hash = hashlib.sha256() + with open(file_path, 'rb') as f: + fb = f.read(BLOCK_SIZE) + while len(fb) > 0: + file_hash.update(fb) + fb = f.read(BLOCK_SIZE) + + return file_hash.hexdigest() + + +def rename_files_with_hash(example_pages): files_to_rename = list(example_pages.keys()) folder = HASHED_PDFS_FOR_TESTING @@ -88,9 +85,9 @@ def rename_files_with_hash(example_pages, hashes): only_name = path.splitext(file)[0] # Adding the new name with extension - new_base = only_name + '_new' + '.txt' + hash = find_hash(old_name) # construct full file path - new_name = path.join(folder, new_base) + new_name = path.join(folder, hash + ".pdf") # Renaming the file os.rename(old_name, new_name) @@ -99,17 +96,12 @@ def rename_files_with_hash(example_pages, hashes): res = os.listdir(folder) print(res) -def hash_pdfs(example_pages): - pdf_paths = list(path.join(PDF_FOR_TESTING, pdf_name) for pdf_name in example_pages.keys()) - hashes = hash_pdf_files(paths=pdf_paths, verbose=0) - example_pages = dict(zip(hashes, example_pages.values())) - return example_pages def main(): examples_pages = read_json(path.join(TEST_DATA_DIR, "example_pages.json")) - # examples_pages = hash_pdfs(examples_pages) - collect_metadata(examples_pages, save=True) - split_pdf(examples_pages) + rename_files_with_hash(examples_pages) + #collect_metadata(examples_pages, save=True) + #split_pdf(examples_pages) if __name__ == "__main__":