From 420e4848963b07ee32983502e9a76c507941e7cc Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 12 Apr 2022 16:48:29 +0200 Subject: [PATCH 01/11] the thresholds deciding weather a countour is likely a primary text structure can be set better, as text structures are not always removed. this leads to over detection of figures --- cv_analysis/figure_detection.py | 1 + cv_analysis/utils/text.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index c4b4b30..3827536 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -19,6 +19,7 @@ def detect_figures(image: np.array): image = image.copy() image = remove_primary_text_regions(image) + show_mpl(image) cnts = detect_large_coherent_structures(image) cnts = filter(is_likely_figure, cnts) diff --git a/cv_analysis/utils/text.py b/cv_analysis/utils/text.py index acfaa48..7ce6d7f 100644 --- a/cv_analysis/utils/text.py +++ b/cv_analysis/utils/text.py @@ -36,7 +36,7 @@ def find_primary_text_regions(image): """ def is_likely_primary_text_segments(cnt): - return 800 < cv2.contourArea(cnt) < 15000 + return 700 < cv2.contourArea(cnt) < 16000 image = image.copy() @@ -47,12 +47,11 @@ def find_primary_text_regions(image): close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (20, 3)) close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=1) - + # show_mpl(close) dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 3)) dilate = cv2.dilate(close, dilate_kernel, iterations=1) - + # show_mpl(dilate) cnts, _ = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - cnts = filter(is_likely_primary_text_segments, cnts) return cnts From 3669b6b341361738437aecc46835ee73ac9b53f1 Mon Sep 17 00:00:00 2001 From: llocarnini Date: Wed, 20 Apr 2022 09:43:30 +0200 Subject: [PATCH 02/11] fig_detection_with_layout.py: approach to label the content of a page through layout detection, table parsing for detected tables needs to be added and overall codes needs to be reviewed layout_parsing.py added condition so fig_detection_with_layout.py works table_parsing.py uncommented line for better table parsing text.py changed kernel sizes --- cv_analysis/fig_detection_with_layout.py | 59 ++++++++++++++++++++++++ cv_analysis/figure_detection.py | 15 ++++-- cv_analysis/layout_parsing.py | 9 ++-- cv_analysis/table_parsing.py | 11 ++++- cv_analysis/utils/text.py | 16 ++++--- 5 files changed, 96 insertions(+), 14 deletions(-) create mode 100644 cv_analysis/fig_detection_with_layout.py diff --git a/cv_analysis/fig_detection_with_layout.py b/cv_analysis/fig_detection_with_layout.py new file mode 100644 index 0000000..bd84789 --- /dev/null +++ b/cv_analysis/fig_detection_with_layout.py @@ -0,0 +1,59 @@ +from cv_analysis.layout_parsing import annotate_layout_in_pdf +from cv_analysis.figure_detection import figures_in_image, detect_figures +from cv_analysis.table_parsing import tables_in_image +from cv_analysis.utils.text import find_primary_text_regions, remove_primary_text_regions +from cv_analysis.utils.draw import draw_rectangles +from cv_analysis.utils.display import show_mpl + + +def detect_parting_line(image): + pass + + +def cut_out_content_structures(layout_rects, page): + large_enough_rects = [] + too_small_rects = [] + for x, y, w, h in layout_rects: + rect = (x, y, w, h) + if w * h >= 100000: + cropped_page = page[y:(y + h), x:(x + w)] + large_enough_rects.append([rect, cropped_page]) + else: + cropped_page = page[y:(y + h), x:(x + w)] + too_small_rects.append([rect, cropped_page]) + return large_enough_rects, too_small_rects + + +def parse_and_label_content_structures(page, large_enough_rects, too_small_rects): + for coordinates, cropped_image in large_enough_rects: + non_text_rects = detect_figures(cropped_image) + print(len(non_text_rects), len(list(non_text_rects))) + if len(non_text_rects) == 0: + page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) + elif tables_in_image(cropped_image)[0]: + page = draw_rectangles(page, [coordinates], color=(255, 0, 0), annotate=True) + else: + page = draw_rectangles(page, [coordinates], color=(0, 0, 255), annotate=True) + + # for coordinates, cropped_image in too_small_rects: + # non_text_rects = detect_figures(cropped_image) + # if len(non_text_rects) == 0 and len(list(find_primary_text_regions(cropped_image))) > 0: + # page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) + # else: + # page = draw_rectangles(page, [coordinates], color=(0, 255, 255), annotate=True) + return page + + +def detect_figures_over_layout(): + # pdf_path = "/home/lillian/PycharmProjects/ner_address/data/pdfs/syngenta/026c917f04660aaea4bb57d180f9598b.pdf" + # pdf_path = "/home/lillian/ocr_docs/ocr1.pdf" + pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" + #pdf_path = "/home/lillian/ocr_docs/VV-857853.pdf" + page_index = 13 + layout_rects, page = annotate_layout_in_pdf(pdf_path, page_index, return_rects=True) + big_structures, small_structures = cut_out_content_structures(layout_rects, page) + page = parse_and_label_content_structures(page, big_structures, small_structures) + show_mpl(page) + + +detect_figures_over_layout() diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index 3827536..4c44a57 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -15,11 +15,10 @@ def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6): def detect_figures(image: np.array): - image = image.copy() - + #show_mpl(image) image = remove_primary_text_regions(image) - show_mpl(image) + #show_mpl(image) cnts = detect_large_coherent_structures(image) cnts = filter(is_likely_figure, cnts) @@ -30,7 +29,6 @@ def detect_figures(image: np.array): def detect_figures_in_pdf(pdf_path, page_index=1, show=False): - page = pdf2image.convert_from_path(pdf_path, first_page=page_index + 1, last_page=page_index + 1)[0] page = np.array(page) @@ -41,3 +39,12 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): show_mpl(page) else: return page + + +def figures_in_image(cropped_page): + redaction_contours = detect_figures(cropped_page) + + if len(redaction_contours) > 0: + return True + else: + return False diff --git a/cv_analysis/layout_parsing.py b/cv_analysis/layout_parsing.py index ae5559f..19d56a1 100644 --- a/cv_analysis/layout_parsing.py +++ b/cv_analysis/layout_parsing.py @@ -63,15 +63,18 @@ def parse_layout(image: np.array): return list(rects) -def annotate_layout_in_pdf(pdf_path, page_index=1, show=False): +def annotate_layout_in_pdf(pdf_path, page_index=1, return_rects=False, show=False): page = pdf2image.convert_from_path(pdf_path, first_page=page_index + 1, last_page=page_index + 1)[0] page = np.array(page) rects = parse_layout(page) - page = draw_rectangles(page, rects) - if show: + if return_rects: + return rects, page + elif show: + page = draw_rectangles(page, rects) show_mpl(page) else: + page = draw_rectangles(page, rects) return page diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index 404b7ed..83dd2e5 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -10,13 +10,14 @@ from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d, remove_isolated from cv_analysis.utils.deskew import deskew_histbased +from cv_analysis.utils.filters import is_large_enough from cv_analysis.layout_parsing import parse_layout def add_external_contours(image, img): contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - # contours = filter(partial(is_large_enough, min_area=5000000), contours) + contours = filter(partial(is_large_enough, min_area=5000000), contours) for cnt in contours: x, y, w, h = cv2.boundingRect(cnt) @@ -154,3 +155,11 @@ def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False): show_mpl(page) else: return page + +def tables_in_image(cropped_image): + table_rects = parse_table(cropped_image) + + if len(table_rects)>0: + return True, table_rects + else: + return False, None diff --git a/cv_analysis/utils/text.py b/cv_analysis/utils/text.py index 7ce6d7f..31f3d2c 100644 --- a/cv_analysis/utils/text.py +++ b/cv_analysis/utils/text.py @@ -1,5 +1,5 @@ import cv2 - +from cv_analysis.utils.display import show_mpl def remove_primary_text_regions(image): """Removes regions of primary text, meaning no figure descriptions for example, but main text body paragraphs. @@ -17,6 +17,7 @@ def remove_primary_text_regions(image): for cnt in cnts: x, y, w, h = cv2.boundingRect(cnt) + print(x,y,w,h, w*h, w/h) cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), -1) return image @@ -36,7 +37,9 @@ def find_primary_text_regions(image): """ def is_likely_primary_text_segments(cnt): - return 700 < cv2.contourArea(cnt) < 16000 + x,y,w,h = cv2.boundingRect(cnt) + print(cv2.contourArea(cnt)) + return 800 < cv2.contourArea(cnt) < 16000 or w/h > 3 image = image.copy() @@ -45,13 +48,14 @@ def find_primary_text_regions(image): image = cv2.threshold(image, 253, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] - close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (20, 3)) + close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (17, 7)) close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=1) - # show_mpl(close) - dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 3)) + show_mpl(close) + dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 5)) dilate = cv2.dilate(close, dilate_kernel, iterations=1) - # show_mpl(dilate) + show_mpl(dilate) cnts, _ = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) cnts = filter(is_likely_primary_text_segments, cnts) return cnts + From 11a24657895d4d2cc04d38d813754df86c08612d Mon Sep 17 00:00:00 2001 From: llocarnini Date: Fri, 22 Apr 2022 10:12:28 +0200 Subject: [PATCH 03/11] few corrections for including smaller figures --- cv_analysis/fig_detection_with_layout.py | 47 +++++++++++------------- cv_analysis/figure_detection.py | 23 ++++++++---- cv_analysis/table_parsing.py | 3 +- cv_analysis/utils/text.py | 19 +++++----- scripts/annotate.py | 5 ++- 5 files changed, 53 insertions(+), 44 deletions(-) diff --git a/cv_analysis/fig_detection_with_layout.py b/cv_analysis/fig_detection_with_layout.py index bd84789..921c256 100644 --- a/cv_analysis/fig_detection_with_layout.py +++ b/cv_analysis/fig_detection_with_layout.py @@ -1,41 +1,41 @@ from cv_analysis.layout_parsing import annotate_layout_in_pdf -from cv_analysis.figure_detection import figures_in_image, detect_figures -from cv_analysis.table_parsing import tables_in_image +from cv_analysis.figure_detection import detect_figures +from cv_analysis.table_parsing import tables_in_image, parse_table from cv_analysis.utils.text import find_primary_text_regions, remove_primary_text_regions from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.display import show_mpl -def detect_parting_line(image): - pass - - def cut_out_content_structures(layout_rects, page): - large_enough_rects = [] - too_small_rects = [] + large_rects = [] + small_rects = [] for x, y, w, h in layout_rects: rect = (x, y, w, h) - if w * h >= 100000: + if w * h >= 50000: cropped_page = page[y:(y + h), x:(x + w)] - large_enough_rects.append([rect, cropped_page]) + large_rects.append([rect, cropped_page]) else: cropped_page = page[y:(y + h), x:(x + w)] - too_small_rects.append([rect, cropped_page]) - return large_enough_rects, too_small_rects + small_rects.append([rect, cropped_page]) + return large_rects, small_rects -def parse_and_label_content_structures(page, large_enough_rects, too_small_rects): - for coordinates, cropped_image in large_enough_rects: +def parse_content_structures(page, large_rects, small_rects): + for coordinates, cropped_image in large_rects: non_text_rects = detect_figures(cropped_image) - print(len(non_text_rects), len(list(non_text_rects))) + if len(non_text_rects) == 0: page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) + elif tables_in_image(cropped_image)[0]: page = draw_rectangles(page, [coordinates], color=(255, 0, 0), annotate=True) + stats = parse_table(page) + page = draw_rectangles(page, stats, annotate=True) + else: page = draw_rectangles(page, [coordinates], color=(0, 0, 255), annotate=True) - # for coordinates, cropped_image in too_small_rects: + # for coordinates, cropped_image in small_rects: # non_text_rects = detect_figures(cropped_image) # if len(non_text_rects) == 0 and len(list(find_primary_text_regions(cropped_image))) > 0: # page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) @@ -44,16 +44,13 @@ def parse_and_label_content_structures(page, large_enough_rects, too_small_rects return page -def detect_figures_over_layout(): - # pdf_path = "/home/lillian/PycharmProjects/ner_address/data/pdfs/syngenta/026c917f04660aaea4bb57d180f9598b.pdf" - # pdf_path = "/home/lillian/ocr_docs/ocr1.pdf" - pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" - #pdf_path = "/home/lillian/ocr_docs/VV-857853.pdf" - page_index = 13 +def detect_figures_with_layout_parsing(pdf_path, page_index=1, show=False): layout_rects, page = annotate_layout_in_pdf(pdf_path, page_index, return_rects=True) big_structures, small_structures = cut_out_content_structures(layout_rects, page) - page = parse_and_label_content_structures(page, big_structures, small_structures) - show_mpl(page) + page = parse_content_structures(page, big_structures, small_structures) + if show: + show_mpl(page) + else: + return page -detect_figures_over_layout() diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index 4c44a57..ab43883 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -1,6 +1,7 @@ import cv2 import numpy as np from pdf2image import pdf2image +from PIL import Image from cv_analysis.utils.detection import detect_large_coherent_structures from cv_analysis.utils.display import show_mpl @@ -16,9 +17,7 @@ def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6): def detect_figures(image: np.array): image = image.copy() - #show_mpl(image) image = remove_primary_text_regions(image) - #show_mpl(image) cnts = detect_large_coherent_structures(image) cnts = filter(is_likely_figure, cnts) @@ -41,10 +40,18 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): return page -def figures_in_image(cropped_page): - redaction_contours = detect_figures(cropped_page) - if len(redaction_contours) > 0: - return True - else: - return False +# pages = [] +# for i in range(0,16): +# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" +# page_index = i +# page = detect_figures_in_pdf(pdf_path, page_index, show=False) +# pages.append(Image.fromarray(page)) +# p1, p = pages[0], pages[1:] +# +# out_pdf_path = "/home/lillian/ocr_docs/out.pdf" +# +# p1.save( +# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p +# ) + diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index 83dd2e5..f6b3286 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -156,10 +156,11 @@ def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False): else: return page + def tables_in_image(cropped_image): table_rects = parse_table(cropped_image) - if len(table_rects)>0: + if len(table_rects) > 0: return True, table_rects else: return False, None diff --git a/cv_analysis/utils/text.py b/cv_analysis/utils/text.py index 31f3d2c..6161db2 100644 --- a/cv_analysis/utils/text.py +++ b/cv_analysis/utils/text.py @@ -1,4 +1,6 @@ import cv2 +import numpy as np + from cv_analysis.utils.display import show_mpl def remove_primary_text_regions(image): @@ -14,12 +16,9 @@ def remove_primary_text_regions(image): image = image.copy() cnts = find_primary_text_regions(image) - for cnt in cnts: x, y, w, h = cv2.boundingRect(cnt) - print(x,y,w,h, w*h, w/h) cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), -1) - return image @@ -38,7 +37,6 @@ def find_primary_text_regions(image): def is_likely_primary_text_segments(cnt): x,y,w,h = cv2.boundingRect(cnt) - print(cv2.contourArea(cnt)) return 800 < cv2.contourArea(cnt) < 16000 or w/h > 3 image = image.copy() @@ -48,14 +46,17 @@ def find_primary_text_regions(image): image = cv2.threshold(image, 253, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] - close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (17, 7)) + close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (17, 7)) #20,3 close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=1) - show_mpl(close) - dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (7, 5)) + + #show_mpl(close) + + dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(7, 4)) #5,3 dilate = cv2.dilate(close, dilate_kernel, iterations=1) - show_mpl(dilate) + + #show_mpl(dilate) + cnts, _ = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) cnts = filter(is_likely_primary_text_segments, cnts) return cnts - diff --git a/scripts/annotate.py b/scripts/annotate.py index 03ab3db..e9eae7d 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -4,13 +4,14 @@ from cv_analysis.table_parsing import annotate_tables_in_pdf from cv_analysis.redaction_detection import annotate_redactions_in_pdf from cv_analysis.layout_parsing import annotate_layout_in_pdf from cv_analysis.figure_detection import detect_figures_in_pdf +from cv_analysis.fig_detection_with_layout import detect_figures_with_layout_parsing def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("pdf_path") parser.add_argument("page_index", type=int) - parser.add_argument("--type", choices=["table", "redaction", "layout", "figure"]) + parser.add_argument("--type", choices=["table", "redaction", "layout", "figure", "figure2"]) args = parser.parse_args() @@ -27,3 +28,5 @@ if __name__ == "__main__": annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=True) elif args.type == "figure": detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + elif args.type == "figure2": + detect_figures_with_layout_parsing(args.pdf_path, page_index=args.page_index, show=True) From 19fe6965fb3fb583e93825e810e87c17957e32fc Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 26 Apr 2022 11:19:27 +0200 Subject: [PATCH 04/11] added line in display so the visual logger doesn't open too many plots changes to fig_detection_with_layout.py so tables are getting parsed as well reusage of adding external contour in table_parsing.py --- cv_analysis/fig_detection_with_layout.py | 40 ++++++++++++++++-------- cv_analysis/figure_detection.py | 14 --------- cv_analysis/layout_parsing.py | 18 +++++------ cv_analysis/table_parsing.py | 24 +++++++------- cv_analysis/utils/display.py | 1 + 5 files changed, 47 insertions(+), 50 deletions(-) diff --git a/cv_analysis/fig_detection_with_layout.py b/cv_analysis/fig_detection_with_layout.py index 921c256..a9226f9 100644 --- a/cv_analysis/fig_detection_with_layout.py +++ b/cv_analysis/fig_detection_with_layout.py @@ -4,6 +4,9 @@ from cv_analysis.table_parsing import tables_in_image, parse_table from cv_analysis.utils.text import find_primary_text_regions, remove_primary_text_regions from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.display import show_mpl +from cv_analysis.utils.visual_logging import vizlogger +from PIL import Image + def cut_out_content_structures(layout_rects, page): @@ -11,7 +14,7 @@ def cut_out_content_structures(layout_rects, page): small_rects = [] for x, y, w, h in layout_rects: rect = (x, y, w, h) - if w * h >= 50000: + if w * h >= 75000: cropped_page = page[y:(y + h), x:(x + w)] large_rects.append([rect, cropped_page]) else: @@ -22,22 +25,18 @@ def cut_out_content_structures(layout_rects, page): def parse_content_structures(page, large_rects, small_rects): for coordinates, cropped_image in large_rects: - non_text_rects = detect_figures(cropped_image) - - if len(non_text_rects) == 0: + figure_rects = detect_figures(cropped_image) + if len(figure_rects) == 0: # text page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) - - elif tables_in_image(cropped_image)[0]: - page = draw_rectangles(page, [coordinates], color=(255, 0, 0), annotate=True) + elif tables_in_image(cropped_image)[0]: # table stats = parse_table(page) - page = draw_rectangles(page, stats, annotate=True) - - else: + page = draw_rectangles(page, stats, color=(255, 0, 0), annotate=True) + else: # figure page = draw_rectangles(page, [coordinates], color=(0, 0, 255), annotate=True) # for coordinates, cropped_image in small_rects: - # non_text_rects = detect_figures(cropped_image) - # if len(non_text_rects) == 0 and len(list(find_primary_text_regions(cropped_image))) > 0: + # figure_rects = detect_figures(cropped_image) + # if len(figure_rects) == 0 and len(list(find_primary_text_regions(cropped_image))) > 0: # page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) # else: # page = draw_rectangles(page, [coordinates], color=(0, 255, 255), annotate=True) @@ -48,9 +47,24 @@ def detect_figures_with_layout_parsing(pdf_path, page_index=1, show=False): layout_rects, page = annotate_layout_in_pdf(pdf_path, page_index, return_rects=True) big_structures, small_structures = cut_out_content_structures(layout_rects, page) page = parse_content_structures(page, big_structures, small_structures) - + vizlogger.debug(page, "figures03_final.png") if show: show_mpl(page) else: return page +# pages = [] +# for i in range(0,16): +# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" +# page_index = i +# layout_rects, page = annotate_layout_in_pdf(pdf_path, page_index, return_rects=True) +# big_structures, small_structures = cut_out_content_structures(layout_rects, page) +# page = parse_content_structures(page, big_structures, small_structures) +# pages.append(Image.fromarray(page)) +# p1, p = pages[0], pages[1:] +# +# out_pdf_path = "/home/lillian/ocr_docs/out1.pdf" +# +# p1.save( +# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p +# ) diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index 72f3543..e1a67b8 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -1,7 +1,6 @@ import cv2 import numpy as np from pdf2image import pdf2image -from PIL import Image from cv_analysis.utils.detection import detect_large_coherent_structures from cv_analysis.utils.display import show_mpl @@ -43,16 +42,3 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): if show: show_mpl(page) -# pages = [] -# for i in range(0,16): -# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" -# page_index = i -# page = detect_figures_in_pdf(pdf_path, page_index, show=False) -# pages.append(Image.fromarray(page)) -# p1, p = pages[0], pages[1:] -# -# out_pdf_path = "/home/lillian/ocr_docs/out.pdf" -# -# p1.save( -# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p -# ) \ No newline at end of file diff --git a/cv_analysis/layout_parsing.py b/cv_analysis/layout_parsing.py index cdc061f..b6ae567 100644 --- a/cv_analysis/layout_parsing.py +++ b/cv_analysis/layout_parsing.py @@ -36,17 +36,17 @@ def parse_layout(image: np.array): if len(image_.shape) > 2: image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2GRAY) - vizlogger.debug(image_, "layout01_start.png") + #vizlogger.debug(image_, "layout01_start.png") image_ = cv2.GaussianBlur(image_, (7, 7), 0) - vizlogger.debug(image_, "layout02_blur.png") + #vizlogger.debug(image_, "layout02_blur.png") thresh = cv2.threshold(image_, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] vizlogger.debug(image_, "layout03_theshold.png") kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) - vizlogger.debug(kernel, "layout04_kernel.png") + #vizlogger.debug(kernel, "layout04_kernel.png") dilate = cv2.dilate(thresh, kernel, iterations=4) - vizlogger.debug(dilate, "layout05_dilate.png") + #vizlogger.debug(dilate, "layout05_dilate.png") rects = list(find_segments(dilate)) @@ -55,16 +55,16 @@ def parse_layout(image: np.array): x, y, w, h = rect cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 0), -1) cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), 7) - vizlogger.debug(image, "layout06_rectangles.png") + #vizlogger.debug(image, "layout06_rectangles.png") _, image = cv2.threshold(image, 254, 255, cv2.THRESH_BINARY) - vizlogger.debug(image, "layout07_threshold.png") + #vizlogger.debug(image, "layout07_threshold.png") image = ~image - vizlogger.debug(image, "layout08_inverse.png") + #vizlogger.debug(image, "layout08_inverse.png") if len(image.shape) > 2: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - vizlogger.debug(image, "layout09_convertcolor.png") + #vizlogger.debug(image, "layout09_convertcolor.png") rects = find_segments(image) # <- End of meta detection @@ -87,8 +87,6 @@ def annotate_layout_in_pdf(pdf_path, page_index=1, return_rects=False, show=Fals elif show: page = draw_rectangles(page, rects) vizlogger.debug(page, "layout10_output.png") - - if show: show_mpl(page) else: page = draw_rectangles(page, rects) diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index 114b41d..d2bd1c3 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -18,8 +18,7 @@ from cv_analysis.layout_parsing import parse_layout def add_external_contours(image, img): contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - - contours = filter(partial(is_large_enough, min_area=5000000), contours) + contours = filter(partial(is_large_enough, min_area=5000), contours) for cnt in contours: x, y, w, h = cv2.boundingRect(cnt) @@ -52,7 +51,7 @@ def apply_motion_blur(image: np.array, angle, size=80): return blurred -def isolate_vertical_and_horizontal_components(img_bin, bounding_rects): +def isolate_vertical_and_horizontal_components(img_bin): """Identifies and reinforces horizontal and vertical lines in a binary image. Args: @@ -69,19 +68,19 @@ def isolate_vertical_and_horizontal_components(img_bin, bounding_rects): img_bin_h = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_h) vizlogger.debug(img_bin_h, "tables01_isolate01_img_bin_h.png") img_bin_v = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_v) - vizlogger.debug(img_bin_v, "tables02_isolate02_img_bin_v.png") + vizlogger.debug(img_bin_v | img_bin_h, "tables02_isolate02_img_bin_v.png") kernel_h = np.ones((1, 30), np.uint8) kernel_v = np.ones((30, 1), np.uint8) img_bin_h = cv2.dilate(img_bin_h, kernel_h, iterations=2) vizlogger.debug(img_bin_h, "tables03_isolate03_dilate_h.png") img_bin_v = cv2.dilate(img_bin_v, kernel_v, iterations=2) - vizlogger.debug(img_bin_v, "tables04_isolate04_dilate_v.png") + vizlogger.debug(img_bin_v | img_bin_h, "tables04_isolate04_dilate_v.png") img_bin_h = apply_motion_blur(img_bin_h, 0) vizlogger.debug(img_bin_h, "tables09_isolate05_blur_h.png") img_bin_v = apply_motion_blur(img_bin_v, 90) - vizlogger.debug(img_bin_v, "tables10_isolate06_blur_v.png") + vizlogger.debug(img_bin_v | img_bin_h, "tables10_isolate06_blur_v.png") img_bin_final = img_bin_h | img_bin_v vizlogger.debug(img_bin_final, "tables11_isolate07_final.png") @@ -91,9 +90,6 @@ def isolate_vertical_and_horizontal_components(img_bin, bounding_rects): img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1) vizlogger.debug(img_bin_final, "tables11_isolate13_dilate.png") - img_bin_final = disconnect_non_existing_cells(img_bin_final, bounding_rects) - vizlogger.debug(img_bin_final, "tables12_isolate14_disconnect.png") - return img_bin_final @@ -160,7 +156,12 @@ def parse_table(image: np.array, show=False): image = preprocess(image) table_layout_boxes = find_table_layout_boxes(image) - image = isolate_vertical_and_horizontal_components(image, table_layout_boxes) + + image = isolate_vertical_and_horizontal_components(image) + image = disconnect_non_existing_cells(image, table_layout_boxes) + vizlogger.debug(image, "tables12_isolate14_disconnect.png") + image = add_external_contours(image, image) + vizlogger.debug(image, "external_contours_added.png") _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) @@ -181,9 +182,6 @@ def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False): stats = parse_table(page) page = draw_rectangles(page, stats, annotate=True) - - if show: - show_mpl(page) vizlogger.debug(page, "tables15_final_output.png") diff --git a/cv_analysis/utils/display.py b/cv_analysis/utils/display.py index 4d346a5..999c9a2 100644 --- a/cv_analysis/utils/display.py +++ b/cv_analysis/utils/display.py @@ -18,6 +18,7 @@ def save_mpl(image, path): ax.imshow(image, cmap="gray") # plt.close() plt.savefig(path) + plt.close() def show_cv2(image): From b806c3c13d73c35db5c38732816453b815ca9eb6 Mon Sep 17 00:00:00 2001 From: Isaac Riley Date: Wed, 27 Apr 2022 09:15:15 +0200 Subject: [PATCH 05/11] fix for table parsing when no outer line is present --- cv_analysis/table_parsing.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index d2bd1c3..0a6ceed 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -16,8 +16,8 @@ from cv_analysis.utils.visual_logging import vizlogger from cv_analysis.layout_parsing import parse_layout -def add_external_contours(image, img): - contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) +def add_external_contours(image, contour_source_image): + contours, _ = cv2.findContours(contour_source_image, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) contours = filter(partial(is_large_enough, min_area=5000), contours) for cnt in contours: @@ -27,6 +27,16 @@ def add_external_contours(image, img): return image +def extend_lines(): + #TODO + pass + + +def make_table_block_mask(): + #TODO + pass + + def apply_motion_blur(image: np.array, angle, size=80): """Solidifies and slightly extends detected lines. @@ -68,8 +78,9 @@ def isolate_vertical_and_horizontal_components(img_bin): img_bin_h = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_h) vizlogger.debug(img_bin_h, "tables01_isolate01_img_bin_h.png") img_bin_v = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_v) - vizlogger.debug(img_bin_v | img_bin_h, "tables02_isolate02_img_bin_v.png") - + img_lines_raw = img_bin_v | img_bin_h + vizlogger.debug(img_lines_raw, "tables02_isolate02_img_bin_v.png") + kernel_h = np.ones((1, 30), np.uint8) kernel_v = np.ones((30, 1), np.uint8) img_bin_h = cv2.dilate(img_bin_h, kernel_h, iterations=2) @@ -89,17 +100,14 @@ def isolate_vertical_and_horizontal_components(img_bin): vizlogger.debug(img_bin_final, "tables10_isolate12_threshold.png") img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1) vizlogger.debug(img_bin_final, "tables11_isolate13_dilate.png") + + # add contours before lines are extended by blurring + img_bin_final = add_external_contours(img_bin_final, img_lines_raw) + vizlogger.debug(img_bin_final, "tables11_isolate14_contours_added.png") return img_bin_final -def disconnect_non_existing_cells(img_bin, bounding_rects): - for rect in bounding_rects: - x, y, w, h = rect - img_bin = cv2.rectangle(img_bin, (x, y), (x + w, y + h), (0, 0, 0), 5) - return img_bin - - def has_table_shape(rects): assert isinstance(rects, list) @@ -158,10 +166,8 @@ def parse_table(image: np.array, show=False): table_layout_boxes = find_table_layout_boxes(image) image = isolate_vertical_and_horizontal_components(image) - image = disconnect_non_existing_cells(image, table_layout_boxes) - vizlogger.debug(image, "tables12_isolate14_disconnect.png") - image = add_external_contours(image, image) - vizlogger.debug(image, "external_contours_added.png") + #image = add_external_contours(image, image) + #vizlogger.debug(image, "external_contours_added.png") _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) From 2c39ffbcdda633da8fbc60681f851a5c817e70b5 Mon Sep 17 00:00:00 2001 From: llocarnini Date: Wed, 27 Apr 2022 11:12:23 +0200 Subject: [PATCH 06/11] changed kernel and iteration for better text removal --- cv_analysis/fig_detection_with_layout.py | 12 +++++++----- cv_analysis/figure_detection.py | 16 +++++++++++++++- cv_analysis/utils/text.py | 7 ++++--- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/cv_analysis/fig_detection_with_layout.py b/cv_analysis/fig_detection_with_layout.py index a9226f9..7f16244 100644 --- a/cv_analysis/fig_detection_with_layout.py +++ b/cv_analysis/fig_detection_with_layout.py @@ -1,11 +1,10 @@ from cv_analysis.layout_parsing import annotate_layout_in_pdf from cv_analysis.figure_detection import detect_figures from cv_analysis.table_parsing import tables_in_image, parse_table -from cv_analysis.utils.text import find_primary_text_regions, remove_primary_text_regions from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.display import show_mpl from cv_analysis.utils.visual_logging import vizlogger -from PIL import Image +#from PIL import Image @@ -28,9 +27,12 @@ def parse_content_structures(page, large_rects, small_rects): figure_rects = detect_figures(cropped_image) if len(figure_rects) == 0: # text page = draw_rectangles(page, [coordinates], color=(0, 255, 0), annotate=True) - elif tables_in_image(cropped_image)[0]: # table - stats = parse_table(page) - page = draw_rectangles(page, stats, color=(255, 0, 0), annotate=True) + elif len(parse_table(cropped_image)) > 0: + #elif tables_in_image(cropped_image)[0]: # table + stats = parse_table(cropped_image) + cropped_image = draw_rectangles(cropped_image, stats, color=(255, 0, 0), annotate=True) + x,y,w,h = coordinates + page[y:y+h, x:x+w] = cropped_image else: # figure page = draw_rectangles(page, [coordinates], color=(0, 0, 255), annotate=True) diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index e1a67b8..f7fdf2b 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -9,7 +9,7 @@ from cv_analysis.utils.post_processing import remove_included from cv_analysis.utils.filters import is_large_enough, has_acceptable_format from cv_analysis.utils.text import remove_primary_text_regions from cv_analysis.utils.visual_logging import vizlogger - +#from PIL import Image def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6): return is_large_enough(cont, min_area) and has_acceptable_format(cont, max_width_to_hight_ratio) @@ -41,4 +41,18 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): vizlogger.debug(page, "figures03_final.png") if show: show_mpl(page) + return page +# pages = [] +# for i in range(0,16): +# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" +# page_index = i +# page = detect_figures_in_pdf(pdf_path,page_index) +# pages.append(Image.fromarray(page)) +# p1, p = pages[0], pages[1:] +# +# out_pdf_path = "/home/lillian/ocr_docs/out.pdf" +# +# p1.save( +# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p +# ) \ No newline at end of file diff --git a/cv_analysis/utils/text.py b/cv_analysis/utils/text.py index 6161db2..01f6c4b 100644 --- a/cv_analysis/utils/text.py +++ b/cv_analysis/utils/text.py @@ -19,6 +19,7 @@ def remove_primary_text_regions(image): for cnt in cnts: x, y, w, h = cv2.boundingRect(cnt) cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), -1) + #show_mpl(image) return image @@ -46,12 +47,12 @@ def find_primary_text_regions(image): image = cv2.threshold(image, 253, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] - close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (17, 7)) #20,3 - close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=1) + close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (20, 3)) #20,3 + close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=2) #show_mpl(close) - dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(7, 4)) #5,3 + dilate_kernel = cv2.getStructuringElement(cv2.MORPH_RECT,(7, 3)) #5,3 dilate = cv2.dilate(close, dilate_kernel, iterations=1) #show_mpl(dilate) From 179ad2016570af7e0a2ac8a6de1bcca8b4777acf Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 17 May 2022 09:17:24 +0200 Subject: [PATCH 07/11] minor changes, refactoring and testfiles added --- .gitignore | 3 +- config.yaml | 2 +- cv_analysis/fig_detection_with_layout.py | 16 +-- cv_analysis/figure_detection.py | 71 ++++++++--- cv_analysis/locations.py | 3 + .../test/scripts/export_example_pages.py | 116 ++++++++++++++++++ cv_analysis/utils/post_processing.py | 2 +- cv_analysis/utils/text.py | 4 +- data/.gitignore | 6 + data/pdfs_for_testing.dvc | 5 + data/pngs_for_testing.dvc | 5 + scripts/annotate.py | 10 +- 12 files changed, 204 insertions(+), 39 deletions(-) create mode 100644 cv_analysis/test/scripts/export_example_pages.py create mode 100644 data/pdfs_for_testing.dvc create mode 100644 data/pngs_for_testing.dvc diff --git a/.gitignore b/.gitignore index f3b659b..5360d78 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ build_venv/ /.idea/table_parsing.iml /.idea/vcs.xml /results/ -/data /table_parsing.egg-info /target/ /tests/ @@ -22,3 +21,5 @@ build_venv/ /cv_analysis.egg-info/SOURCES.txt /cv_analysis.egg-info/top_level.txt /.vscode/ +/cv_analysis/test/test_data/example_pages.json +/data/metadata_testing_files.csv diff --git a/config.yaml b/config.yaml index 42bd2e7..fc6bb42 100644 --- a/config.yaml +++ b/config.yaml @@ -23,5 +23,5 @@ deskew: test_dummy: test_dummy visual_logging: - level: $LOGGING_LEVEL_ROOT|INFO + level: $LOGGING_LEVEL_ROOT|DEBUG output_folder: /tmp/debug/ \ No newline at end of file diff --git a/cv_analysis/fig_detection_with_layout.py b/cv_analysis/fig_detection_with_layout.py index 7f16244..ce1d71b 100644 --- a/cv_analysis/fig_detection_with_layout.py +++ b/cv_analysis/fig_detection_with_layout.py @@ -55,18 +55,4 @@ def detect_figures_with_layout_parsing(pdf_path, page_index=1, show=False): else: return page -# pages = [] -# for i in range(0,16): -# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" -# page_index = i -# layout_rects, page = annotate_layout_in_pdf(pdf_path, page_index, return_rects=True) -# big_structures, small_structures = cut_out_content_structures(layout_rects, page) -# page = parse_content_structures(page, big_structures, small_structures) -# pages.append(Image.fromarray(page)) -# p1, p = pages[0], pages[1:] -# -# out_pdf_path = "/home/lillian/ocr_docs/out1.pdf" -# -# p1.save( -# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p -# ) + diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index a7db151..468500f 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -1,6 +1,9 @@ import cv2 import numpy as np from pdf2image import pdf2image +import pandas as pd +from PIL import Image +import timeit from cv_analysis.utils.detection import detect_large_coherent_structures from cv_analysis.utils.display import show_mpl @@ -33,7 +36,7 @@ def detect_figures(image: np.array): def detect_figures_in_pdf(pdf_path, page_index=1, show=False): - page = pdf2image.convert_from_path(pdf_path, first_page=page_index + 1, last_page=page_index + 1)[0] + page = pdf2image.convert_from_path(pdf_path, dpi=300, first_page=page_index + 1, last_page=page_index + 1)[0] page = np.array(page) redaction_contours = detect_figures(page) @@ -43,16 +46,56 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): show_mpl(page) return page -# pages = [] -# for i in range(0,16): -# pdf_path = "/home/lillian/ocr_docs/Report on spectra.pdf" -# page_index = i -# page = detect_figures_in_pdf(pdf_path,page_index) -# pages.append(Image.fromarray(page)) -# p1, p = pages[0], pages[1:] -# -# out_pdf_path = "/home/lillian/ocr_docs/out.pdf" -# -# p1.save( -# out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p -# ) \ No newline at end of file + +def detect_figures_in_test_files(): + def save_as_pdf(pages): + p1, p = pages[0], pages[1:] + out_pdf_path = "/home/lillian/ocr_docs/output_files/fig_detection_pdf.pdf" + p1.save( + out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p + ) + path = "/home/lillian/ocr_docs/" + ex_pages = pd.read_csv(path+"/metadata/metadata2.csv") + pages_detected = [] + + t0 = timeit.default_timer() + for name, page_nr in zip(ex_pages.pdf_name, ex_pages.page): + page = pdf2image.convert_from_path(path + "/original/" + name, dpi=300, first_page=page_nr, last_page=page_nr)[0] + page = np.array(page) + redaction_contours = detect_figures(page) + page = draw_rectangles(page, redaction_contours) + pages_detected.append(Image.fromarray(page)) + print(timeit.default_timer()-t0) + + save_as_pdf(pages_detected) + + +def detect_figures_in_png(pdf_path, show=False): + + page = Image.open(pdf_path) + page = np.array(page) + + redaction_contours = detect_figures(page) + page = draw_rectangles(page, redaction_contours) + vizlogger.debug(page, "figures03_final.png") + if show: + show_mpl(page) + return page + + +def detect_figures_in_test_files_png(): + file_name = pd.read_csv("/home/lillian/ocr_docs/metadata/metadata2.csv") + path = "/home/lillian/ocr_docs/png_example_pages/" + pages = [] + page_index = 0 + t0 = timeit.default_timer() + for name in file_name.image_name: + page = detect_figures_in_png(path+name+".png", page_index, show=False) + pages.append(Image.fromarray(page)) + t1 = timeit.default_timer() + print(t1-t0) + p1, p = pages[0], pages[1:] + out_pdf_path = "/home/lillian/ocr_docs/output_files/fig_detection_png2.pdf" + p1.save( + out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p + ) \ No newline at end of file diff --git a/cv_analysis/locations.py b/cv_analysis/locations.py index 6e56ec1..fee3248 100644 --- a/cv_analysis/locations.py +++ b/cv_analysis/locations.py @@ -10,6 +10,9 @@ CONFIG_FILE = path.join(PACKAGE_ROOT_DIR, "config.yaml") LOG_FILE = "/tmp/log.log" DVC_DATA_DIR = path.join(PACKAGE_ROOT_DIR, "data") +PDF_FOR_TESTING = path.join(DVC_DATA_DIR, "pdfs_for_testing") +PNG_FOR_TESTING = path.join(DVC_DATA_DIR, "pngs_for_testing") +HASHED_PDFS = path.join(PDF_FOR_TESTING, "hashed") TEST_DIR = path.join(MODULE_DIR, "test") TEST_DATA_DIR = path.join(MODULE_DIR, "test", "test_data") diff --git a/cv_analysis/test/scripts/export_example_pages.py b/cv_analysis/test/scripts/export_example_pages.py new file mode 100644 index 0000000..5aff47e --- /dev/null +++ b/cv_analysis/test/scripts/export_example_pages.py @@ -0,0 +1,116 @@ +import os +from os import path +import pandas as pd +from pdf2image import convert_from_path +from itertools import chain +import json +from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS +from cv_analysis.utils.deduplicate_pdfs import hash_pdf_files + +def read_json(path): + with open(path, encoding='utf-8') as file: + data = json.load(file) + return data + + +# def collect_metadata(example_pages, save=False): +# metadata = [] +# i = 0 +# for name, document_sections in example_pages.items(): +# for pages in document_sections: +# span = list(range(pages[0], pages[1] + 1)) +# for page_nr in span: +# metadata.append(["fig_table" + str(i), name, page_nr]) +# i += 1 +# if save: +# df = pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) +# df.to_csv("/exported_files/test_pages.csv") +# else: +# return pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) + + + +def collect_metadata(example_pages, save=False): + metadata = [] + make_metadata_entry = make_metadata_entry_maker() + for name, document_sections in example_pages.items(): + metadata.append(f(name, document_sections, make_metadata_entry)) + metadata = list(chain.from_iterable(metadata)) + if save: + df = pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) + df.to_csv(path.join(DVC_DATA_DIR, "metadata_testing_files.csv")) + else: + return pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) + + +def f(name, document_sections, make_metadata_entry): + for pages in document_sections: + span = list(range(pages[0], pages[1] + 1)) + for page_nr in span: + yield make_metadata_entry(name, page_nr) + + +def make_metadata_entry_maker(): + i = -1 + + def make_metadata_entry(name, page_nr): + nonlocal i + i += 1 + return ["fig_table" + str(i), name, page_nr] + + return make_metadata_entry + + +def split_pdf(example_pages): + dir_path = PDF_FOR_TESTING + i = 0 + for name, document_sections in example_pages.items(): + for pages in document_sections: + images = convert_from_path(pdf_path=path.join(dir_path, name), dpi=300, first_page=pages[0], + last_page=pages[1]) + for image in images: + fp = path.join(PNG_FOR_TESTING, "fig_table" + str(i) + ".png") + image.save(fp=fp, dpi=(300, 300)) + i += 1 + +def rename_files_with_hash(example_pages,hashes): + + files_to_rename = list(example_pages.keys()) + folder = HASHED_PDFS + + # Iterate through the folder + for file in os.listdir(folder): + # Checking if the file is present in the list + if file in files_to_rename: + # construct current name using file name and path + old_name = path.join(folder, file) + # get file name without extension + only_name = path.splitext(file)[0] + + # Adding the new name with extension + new_base = only_name + '_new' + '.txt' + # construct full file path + new_name = path.join(folder, new_base) + + # Renaming the file + os.rename(old_name, new_name) + + # verify the result + res = os.listdir(folder) + print(res) + +def hash_pdfs(example_pages): + pdf_paths = list(path.join(PDF_FOR_TESTING, pdf_name) for pdf_name in example_pages.keys()) + hashes = hash_pdf_files(paths=pdf_paths, verbose=0) + example_pages = dict(zip(hashes, example_pages.values())) + return example_pages + +def main(): + examples_pages = read_json(path.join(TEST_DATA_DIR, "example_pages.json")) + examples_pages = hash_pdfs(examples_pages) + collect_metadata(examples_pages, save=True) + #split_pdf(examples_pages) + + +if __name__ == "__main__": + main() diff --git a/cv_analysis/utils/post_processing.py b/cv_analysis/utils/post_processing.py index 753e091..1749f2d 100644 --- a/cv_analysis/utils/post_processing.py +++ b/cv_analysis/utils/post_processing.py @@ -25,7 +25,7 @@ def remove_included(rectangles): return b.xmin + tol >= a.xmin and b.ymin + tol >= a.ymin and b.xmax - tol <= a.xmax and b.ymax - tol <= a.ymax def is_not_included(rect, rectangles): - return not any(included(r2, rect) for r2 in rectangles if not rect == r2) + return not any(includes(r2, rect) for r2 in rectangles if not rect == r2) rectangles = list(map(xywh_to_vec_rect, rectangles)) rectangles = filter(partial(is_not_included, rectangles=rectangles), rectangles) diff --git a/cv_analysis/utils/text.py b/cv_analysis/utils/text.py index 01f6c4b..5d9ccaa 100644 --- a/cv_analysis/utils/text.py +++ b/cv_analysis/utils/text.py @@ -47,8 +47,8 @@ def find_primary_text_regions(image): image = cv2.threshold(image, 253, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] - close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (20, 3)) #20,3 - close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=2) + close_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (17, 7)) #20,3 + close = cv2.morphologyEx(image, cv2.MORPH_CLOSE, close_kernel, iterations=1) #show_mpl(close) diff --git a/data/.gitignore b/data/.gitignore index 09d8485..7b38b1e 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1 +1,7 @@ /test_pdf.pdf +/pdfs_for_testing +/figure_detection.png +/layout_parsing.png +/redaction_detection.png +/table_parsing.png +/pngs_for_testing diff --git a/data/pdfs_for_testing.dvc b/data/pdfs_for_testing.dvc new file mode 100644 index 0000000..e85e518 --- /dev/null +++ b/data/pdfs_for_testing.dvc @@ -0,0 +1,5 @@ +outs: +- md5: bb0ce084f7ca54583972da71cb87e22c.dir + size: 367181628 + nfiles: 28 + path: pdfs_for_testing diff --git a/data/pngs_for_testing.dvc b/data/pngs_for_testing.dvc new file mode 100644 index 0000000..630eab7 --- /dev/null +++ b/data/pngs_for_testing.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 4fed91116111b47edf1c6f6a67eb84d3.dir + size: 58125058 + nfiles: 230 + path: pngs_for_testing diff --git a/scripts/annotate.py b/scripts/annotate.py index 35310bf..34b007c 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -3,7 +3,7 @@ import argparse from cv_analysis.table_parsing import annotate_tables_in_pdf from cv_analysis.redaction_detection import annotate_redactions_in_pdf from cv_analysis.layout_parsing import annotate_layout_in_pdf -from cv_analysis.figure_detection import detect_figures_in_pdf +from cv_analysis.figure_detection import detect_figures_in_pdf, detect_figures_in_test_files from cv_analysis.fig_detection_with_layout import detect_figures_with_layout_parsing @@ -11,7 +11,7 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("pdf_path") parser.add_argument("page_index", type=int) - parser.add_argument("--type", choices=["table", "redaction", "layout", "figure", "figure2"]) + parser.add_argument("--type", choices=["table", "redaction", "layout", "figure", "figures"]) parser.add_argument("--show", action="store_true", default=False) args = parser.parse_args() @@ -28,6 +28,6 @@ if __name__ == "__main__": elif args.type == "layout": annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) elif args.type == "figure": - detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) - elif args.type == "figure2": - detect_figures_with_layout_parsing(args.pdf_path, page_index=args.page_index, show=args.show) + detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + elif args.type == "figures": + detect_figures_in_test_files() From 3f33ab4f3d838f0da043e3526431701cd4a12c0b Mon Sep 17 00:00:00 2001 From: Isaac Riley Date: Tue, 24 May 2022 08:01:42 +0200 Subject: [PATCH 08/11] resolve a DVC conflict --- data/test_pdf.pdf.dvc | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 data/test_pdf.pdf.dvc diff --git a/data/test_pdf.pdf.dvc b/data/test_pdf.pdf.dvc deleted file mode 100644 index 4eff9a4..0000000 --- a/data/test_pdf.pdf.dvc +++ /dev/null @@ -1,4 +0,0 @@ -outs: -- md5: 60840305e4ddb084aea21976b8b7c49e - size: 6916053 - path: test_pdf.pdf From c4c85ace6d717a1e027c999f4694c241cdbc8f8a Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 24 May 2022 09:31:29 +0200 Subject: [PATCH 09/11] added locations and changed names for test_files --- cv_analysis/figure_detection.py | 13 ++++++------- cv_analysis/locations.py | 6 +++++- cv_analysis/test/scripts/export_example_pages.py | 14 +++++++------- scripts/annotate.py | 4 ++-- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index 468500f..32db8d5 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -4,7 +4,8 @@ from pdf2image import pdf2image import pandas as pd from PIL import Image import timeit - +from os import path +from cv_analysis.locations import METADATA_TESTFILES, PNG_FOR_TESTING, PNG_FIGURES_DETECTED from cv_analysis.utils.detection import detect_large_coherent_structures from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles @@ -84,18 +85,16 @@ def detect_figures_in_png(pdf_path, show=False): def detect_figures_in_test_files_png(): - file_name = pd.read_csv("/home/lillian/ocr_docs/metadata/metadata2.csv") - path = "/home/lillian/ocr_docs/png_example_pages/" + file_name = pd.read_csv(METADATA_TESTFILES) pages = [] - page_index = 0 t0 = timeit.default_timer() for name in file_name.image_name: - page = detect_figures_in_png(path+name+".png", page_index, show=False) + page = detect_figures_in_png(path.join(PNG_FOR_TESTING, name+".png")) pages.append(Image.fromarray(page)) t1 = timeit.default_timer() print(t1-t0) p1, p = pages[0], pages[1:] - out_pdf_path = "/home/lillian/ocr_docs/output_files/fig_detection_png2.pdf" + out_pdf_path = path.join(PNG_FIGURES_DETECTED, "fig_detectes.pdf") p1.save( - out_pdf_path, "PDF", resolution=150.0, save_all=True, append_images=p + out_pdf_path, "PDF", resolution=300.0, save_all=True, append_images=p ) \ No newline at end of file diff --git a/cv_analysis/locations.py b/cv_analysis/locations.py index fee3248..07e15f6 100644 --- a/cv_analysis/locations.py +++ b/cv_analysis/locations.py @@ -12,7 +12,11 @@ LOG_FILE = "/tmp/log.log" DVC_DATA_DIR = path.join(PACKAGE_ROOT_DIR, "data") PDF_FOR_TESTING = path.join(DVC_DATA_DIR, "pdfs_for_testing") PNG_FOR_TESTING = path.join(DVC_DATA_DIR, "pngs_for_testing") -HASHED_PDFS = path.join(PDF_FOR_TESTING, "hashed") +PNG_FIGURES_DETECTED = path.join(PNG_FOR_TESTING, "figures_detected") +PNG_TABLES_DETECTED = path.join(PNG_FOR_TESTING, "tables_detected_by_tp") +HASHED_PDFS_FOR_TESTING = path.join(PDF_FOR_TESTING, "hashed") +METADATA_TESTFILES = path.join(DVC_DATA_DIR, "metadata_testing_files.csv") + TEST_DIR = path.join(MODULE_DIR, "test") TEST_DATA_DIR = path.join(MODULE_DIR, "test", "test_data") diff --git a/cv_analysis/test/scripts/export_example_pages.py b/cv_analysis/test/scripts/export_example_pages.py index 5aff47e..65ffbb0 100644 --- a/cv_analysis/test/scripts/export_example_pages.py +++ b/cv_analysis/test/scripts/export_example_pages.py @@ -4,7 +4,7 @@ import pandas as pd from pdf2image import convert_from_path from itertools import chain import json -from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS +from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS_FOR_TESTING from cv_analysis.utils.deduplicate_pdfs import hash_pdf_files def read_json(path): @@ -56,7 +56,7 @@ def make_metadata_entry_maker(): def make_metadata_entry(name, page_nr): nonlocal i i += 1 - return ["fig_table" + str(i), name, page_nr] + return [f"fig_table{i:0>3}", name, page_nr] return make_metadata_entry @@ -69,14 +69,14 @@ def split_pdf(example_pages): images = convert_from_path(pdf_path=path.join(dir_path, name), dpi=300, first_page=pages[0], last_page=pages[1]) for image in images: - fp = path.join(PNG_FOR_TESTING, "fig_table" + str(i) + ".png") + fp = path.join(PNG_FOR_TESTING, f"fig_table{i:0>3}.png") image.save(fp=fp, dpi=(300, 300)) i += 1 -def rename_files_with_hash(example_pages,hashes): +def rename_files_with_hash(example_pages, hashes): files_to_rename = list(example_pages.keys()) - folder = HASHED_PDFS + folder = HASHED_PDFS_FOR_TESTING # Iterate through the folder for file in os.listdir(folder): @@ -107,9 +107,9 @@ def hash_pdfs(example_pages): def main(): examples_pages = read_json(path.join(TEST_DATA_DIR, "example_pages.json")) - examples_pages = hash_pdfs(examples_pages) + # examples_pages = hash_pdfs(examples_pages) collect_metadata(examples_pages, save=True) - #split_pdf(examples_pages) + split_pdf(examples_pages) if __name__ == "__main__": diff --git a/scripts/annotate.py b/scripts/annotate.py index 34b007c..8410ce2 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -3,7 +3,7 @@ import argparse from cv_analysis.table_parsing import annotate_tables_in_pdf from cv_analysis.redaction_detection import annotate_redactions_in_pdf from cv_analysis.layout_parsing import annotate_layout_in_pdf -from cv_analysis.figure_detection import detect_figures_in_pdf, detect_figures_in_test_files +from cv_analysis.figure_detection import detect_figures_in_pdf, detect_figures_in_test_files_png from cv_analysis.fig_detection_with_layout import detect_figures_with_layout_parsing @@ -30,4 +30,4 @@ if __name__ == "__main__": elif args.type == "figure": detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True) elif args.type == "figures": - detect_figures_in_test_files() + detect_figures_in_test_files_png() From 90dfacab21821a3550b0018465b7960919fedfb3 Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 24 May 2022 09:32:48 +0200 Subject: [PATCH 10/11] deleted function for processing testfiles --- scripts/annotate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scripts/annotate.py b/scripts/annotate.py index 8410ce2..002e245 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -3,8 +3,7 @@ import argparse from cv_analysis.table_parsing import annotate_tables_in_pdf from cv_analysis.redaction_detection import annotate_redactions_in_pdf from cv_analysis.layout_parsing import annotate_layout_in_pdf -from cv_analysis.figure_detection import detect_figures_in_pdf, detect_figures_in_test_files_png -from cv_analysis.fig_detection_with_layout import detect_figures_with_layout_parsing +from cv_analysis.figure_detection import detect_figures_in_pdf def parse_args(): @@ -29,5 +28,3 @@ if __name__ == "__main__": annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) elif args.type == "figure": detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True) - elif args.type == "figures": - detect_figures_in_test_files_png() From f5a75f39498d4dfbd026b7779b33006e851d048c Mon Sep 17 00:00:00 2001 From: llocarnini Date: Tue, 24 May 2022 16:20:52 +0200 Subject: [PATCH 11/11] changes in export_example_pages.py as well as removing unused imports in table_parsing.py --- cv_analysis/table_parsing.py | 17 ++++--- .../test/scripts/export_example_pages.py | 50 ++++++++----------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index dbb8334..2b6344b 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -1,16 +1,19 @@ + from functools import partial from itertools import chain, starmap from operator import attrgetter -from os.path import join + import cv2 import numpy as np + from pdf2image import pdf2image + from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d -from cv_analysis.utils.deskew import deskew_histbased +from cv_analysis.utils.deskew import deskew_histbased, deskew from cv_analysis.utils.filters import is_large_enough from cv_analysis.utils.visual_logging import vizlogger from cv_analysis.layout_parsing import parse_layout @@ -26,12 +29,12 @@ def add_external_contours(image, img): def extend_lines(): - #TODO + # TODO pass def make_table_block_mask(): - #TODO + # TODO pass @@ -164,8 +167,8 @@ def parse_table(image: np.array, show=False): table_layout_boxes = find_table_layout_boxes(image) image = isolate_vertical_and_horizontal_components(image) - #image = add_external_contours(image, image) - #vizlogger.debug(image, "external_contours_added.png") + # image = add_external_contours(image, image) + # vizlogger.debug(image, "external_contours_added.png") _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) @@ -198,3 +201,5 @@ def tables_in_image(cropped_image): return True, table_rects else: return False, None + + diff --git a/cv_analysis/test/scripts/export_example_pages.py b/cv_analysis/test/scripts/export_example_pages.py index 65ffbb0..79dcd04 100644 --- a/cv_analysis/test/scripts/export_example_pages.py +++ b/cv_analysis/test/scripts/export_example_pages.py @@ -1,3 +1,4 @@ +import hashlib import os from os import path import pandas as pd @@ -5,7 +6,7 @@ from pdf2image import convert_from_path from itertools import chain import json from cv_analysis.locations import PDF_FOR_TESTING, TEST_DATA_DIR, PNG_FOR_TESTING, DVC_DATA_DIR, HASHED_PDFS_FOR_TESTING -from cv_analysis.utils.deduplicate_pdfs import hash_pdf_files + def read_json(path): with open(path, encoding='utf-8') as file: @@ -13,23 +14,6 @@ def read_json(path): return data -# def collect_metadata(example_pages, save=False): -# metadata = [] -# i = 0 -# for name, document_sections in example_pages.items(): -# for pages in document_sections: -# span = list(range(pages[0], pages[1] + 1)) -# for page_nr in span: -# metadata.append(["fig_table" + str(i), name, page_nr]) -# i += 1 -# if save: -# df = pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) -# df.to_csv("/exported_files/test_pages.csv") -# else: -# return pd.DataFrame(data=metadata, columns=["image_name", "pdf_name", "page"]) - - - def collect_metadata(example_pages, save=False): metadata = [] make_metadata_entry = make_metadata_entry_maker() @@ -73,8 +57,21 @@ def split_pdf(example_pages): image.save(fp=fp, dpi=(300, 300)) i += 1 -def rename_files_with_hash(example_pages, hashes): +def find_hash(file_path): + BLOCK_SIZE = 65536 + + file_hash = hashlib.sha256() + with open(file_path, 'rb') as f: + fb = f.read(BLOCK_SIZE) + while len(fb) > 0: + file_hash.update(fb) + fb = f.read(BLOCK_SIZE) + + return file_hash.hexdigest() + + +def rename_files_with_hash(example_pages): files_to_rename = list(example_pages.keys()) folder = HASHED_PDFS_FOR_TESTING @@ -88,9 +85,9 @@ def rename_files_with_hash(example_pages, hashes): only_name = path.splitext(file)[0] # Adding the new name with extension - new_base = only_name + '_new' + '.txt' + hash = find_hash(old_name) # construct full file path - new_name = path.join(folder, new_base) + new_name = path.join(folder, hash + ".pdf") # Renaming the file os.rename(old_name, new_name) @@ -99,17 +96,12 @@ def rename_files_with_hash(example_pages, hashes): res = os.listdir(folder) print(res) -def hash_pdfs(example_pages): - pdf_paths = list(path.join(PDF_FOR_TESTING, pdf_name) for pdf_name in example_pages.keys()) - hashes = hash_pdf_files(paths=pdf_paths, verbose=0) - example_pages = dict(zip(hashes, example_pages.values())) - return example_pages def main(): examples_pages = read_json(path.join(TEST_DATA_DIR, "example_pages.json")) - # examples_pages = hash_pdfs(examples_pages) - collect_metadata(examples_pages, save=True) - split_pdf(examples_pages) + rename_files_with_hash(examples_pages) + #collect_metadata(examples_pages, save=True) + #split_pdf(examples_pages) if __name__ == "__main__":