From 88bb8dbddfa5af1954d3467ad8776019bab9a70b Mon Sep 17 00:00:00 2001 From: Isaac Riley Date: Thu, 21 Apr 2022 15:10:35 +0200 Subject: [PATCH] added visual logger for development --- config.yaml | 6 +- cv_analysis/figure_detection.py | 8 ++- cv_analysis/layout_parsing.py | 19 ++++-- cv_analysis/redaction_detection.py | 10 ++- cv_analysis/table_parsing.py | 102 ++++++++++++++++------------ cv_analysis/utils/display.py | 11 +++ cv_analysis/utils/visual_logging.py | 22 ++++++ scripts/annotate.py | 10 +-- scripts/client_mock.py | 8 +-- src/run_service.py | 1 + 10 files changed, 133 insertions(+), 64 deletions(-) create mode 100644 cv_analysis/utils/visual_logging.py diff --git a/config.yaml b/config.yaml index 95a70a3..fc6bb42 100644 --- a/config.yaml +++ b/config.yaml @@ -20,4 +20,8 @@ deskew: verbose: False filter_strength_h: 3 -test_dummy: test_dummy \ No newline at end of file +test_dummy: test_dummy + +visual_logging: + level: $LOGGING_LEVEL_ROOT|DEBUG + output_folder: /tmp/debug/ \ No newline at end of file diff --git a/cv_analysis/figure_detection.py b/cv_analysis/figure_detection.py index c4b4b30..4d50233 100644 --- a/cv_analysis/figure_detection.py +++ b/cv_analysis/figure_detection.py @@ -8,6 +8,7 @@ from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.post_processing import remove_included from cv_analysis.utils.filters import is_large_enough, has_acceptable_format from cv_analysis.utils.text import remove_primary_text_regions +from cv_analysis.utils.visual_logging import vizlogger def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6): @@ -17,8 +18,10 @@ def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6): def detect_figures(image: np.array): image = image.copy() + vizlogger.debug(image, "figures01_start.png") image = remove_primary_text_regions(image) + vizlogger.debug(image, "figures02_remove_text.png") cnts = detect_large_coherent_structures(image) cnts = filter(is_likely_figure, cnts) @@ -35,8 +38,7 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False): redaction_contours = detect_figures(page) page = draw_rectangles(page, redaction_contours) - + vizlogger.debug(page, "figures03_final.png") if show: show_mpl(page) - else: - return page + \ No newline at end of file diff --git a/cv_analysis/layout_parsing.py b/cv_analysis/layout_parsing.py index c028296..2d6dcd3 100644 --- a/cv_analysis/layout_parsing.py +++ b/cv_analysis/layout_parsing.py @@ -9,6 +9,7 @@ from pdf2image import pdf2image from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles from cv_analysis.utils.post_processing import remove_overlapping, remove_included, has_no_parent +from cv_analysis.utils.visual_logging import vizlogger def is_likely_segment(rect, min_area=100): @@ -35,11 +36,17 @@ def parse_layout(image: np.array): if len(image_.shape) > 2: image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2GRAY) + vizlogger.debug(image_, "layout01_start.png") + image_ = cv2.GaussianBlur(image_, (7, 7), 0) + vizlogger.debug(image_, "layout02_blur.png") thresh = cv2.threshold(image_, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1] + vizlogger.debug(image_, "layout03_theshold.png") kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) + vizlogger.debug(kernel, "layout04_kernel.png") dilate = cv2.dilate(thresh, kernel, iterations=4) + vizlogger.debug(dilate, "layout05_dilate.png") rects = list(find_segments(dilate)) @@ -48,12 +55,17 @@ def parse_layout(image: np.array): x, y, w, h = rect cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 0), -1) cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), 7) + vizlogger.debug(image, "layout06_rectangles.png") _, image = cv2.threshold(image, 254, 255, cv2.THRESH_BINARY) + vizlogger.debug(image, "layout07_threshold.png") image = ~image + vizlogger.debug(image, "layout08_inverse.png") if len(image.shape) > 2: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + vizlogger.debug(image, "layout09_convertcolor.png") + rects = find_segments(image) # <- End of meta detection @@ -70,12 +82,11 @@ def annotate_layout_in_pdf(pdf_path, page_index=1, show=False): rects = parse_layout(page) page = draw_rectangles(page, rects) + vizlogger.debug(page, "layout10_output.png") if show: show_mpl(page) - else: - return page - + """ def find_layout_boxes(image: np.array): @@ -125,4 +136,4 @@ def annotate_layout_in_pdf(pdf_path, page_index=1): ax.imshow(page) plt.show() -""" \ No newline at end of file +""" diff --git a/cv_analysis/redaction_detection.py b/cv_analysis/redaction_detection.py index c58e106..f4fe7ca 100644 --- a/cv_analysis/redaction_detection.py +++ b/cv_analysis/redaction_detection.py @@ -8,6 +8,7 @@ from iteration_utilities import starfilter, first from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_contours from cv_analysis.utils.filters import is_large_enough, is_filled, is_boxy +from cv_analysis.utils.visual_logging import vizlogger def is_likely_redaction(contour, hierarchy, min_area): @@ -15,15 +16,18 @@ def is_likely_redaction(contour, hierarchy, min_area): def find_redactions(image: np.array, min_normalized_area=200000): - + vizlogger.debug(image, "redactions01_start.png") min_normalized_area /= 200 # Assumes 200 DPI PDF -> image conversion resolution if len(image.shape) > 2: gray = ~cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) else: gray = ~image + vizlogger.debug(gray, "redactions02_gray.png") blurred = cv2.GaussianBlur(gray, (5, 5), 1) + vizlogger.debug(blurred, "redactions03_blur.png") thresh = cv2.threshold(blurred, 252, 255, cv2.THRESH_BINARY)[1] + vizlogger.debug(blurred, "redactions04_threshold.png") contours, hierarchies = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) @@ -43,8 +47,8 @@ def annotate_redactions_in_pdf(pdf_path, page_index=1, show=False): redaction_contours = find_redactions(page) page = draw_contours(page, redaction_contours) + vizlogger.debug(page, "redactions05_output.png") if show: show_mpl(page) - else: - return page + \ No newline at end of file diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index 5a68454..a6a6afd 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -2,81 +2,104 @@ from functools import partial from itertools import chain, starmap from operator import attrgetter +from os.path import join import cv2 import numpy as np from pdf2image import pdf2image from cv_analysis.utils.display import show_mpl from cv_analysis.utils.draw import draw_rectangles -from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d, remove_isolated +from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d from cv_analysis.utils.deskew import deskew_histbased +from cv_analysis.utils.visual_logging import vizlogger from cv_analysis.layout_parsing import parse_layout def add_external_contours(image, img): contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - # contours = filter(partial(is_large_enough, min_area=5000000), contours) for cnt in contours: x, y, w, h = cv2.boundingRect(cnt) cv2.rectangle(image, (x, y), (x + w, y + h), 255, 1) + vizlogger.debug(image, "external_contours.png") return image -def apply_motion_blur(image, size, angle): - """ +def apply_motion_blur(image: np.array, angle, size=80): + """Solidifies and slightly extends detected lines. + + Args: + image (np.array): page image as array + angle: direction in which to apply blur, 0 or 90 + size (int): kernel size; 80 found empirically to work well + + Returns: + np.array """ k = np.zeros((size, size), dtype=np.float32) + vizlogger.debug(k, "tables08_blur_kernel1.png") k[(size - 1) // 2, :] = np.ones(size, dtype=np.float32) + vizlogger.debug(k, "tables09_blur_kernel2.png") k = cv2.warpAffine(k, cv2.getRotationMatrix2D((size / 2 - 0.5, size / 2 - 0.5), angle, 1.0), (size, size)) + vizlogger.debug(k, "tables10_blur_kernel3.png") k = k * (1.0 / np.sum(k)) - return cv2.filter2D(image, -1, k) + vizlogger.debug(k, "tables11_blur_kernel4.png") + blurred = cv2.filter2D(image, -1, k) + return blurred -def isolate_vertical_and_horizontal_components(img_bin, bounding_rects, show=False): +def isolate_vertical_and_horizontal_components(img_bin, bounding_rects): + """Identifies and reinforces horizontal and vertical lines in a binary image. + + Args: + img_bin (np.array): array corresponding to single binarized page image + bounding_rects (list): list of layout boxes of the form (x, y, w, h), potentially containing tables + + Returns: + np.array + """ line_min_width = 48 kernel_h = np.ones((1, line_min_width), np.uint8) kernel_v = np.ones((line_min_width, 1), np.uint8) img_bin_h = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_h) + vizlogger.debug(img_bin_h, "tables01_isolate01_img_bin_h.png") img_bin_v = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_v) - if show: - show_mpl(img_bin_h | img_bin_v) + vizlogger.debug(img_bin_v, "tables02_isolate02_img_bin_v.png") kernel_h = np.ones((1, 30), np.uint8) kernel_v = np.ones((30, 1), np.uint8) img_bin_h = cv2.dilate(img_bin_h, kernel_h, iterations=2) + vizlogger.debug(img_bin_h, "tables03_isolate03_dilate_h.png") img_bin_v = cv2.dilate(img_bin_v, kernel_v, iterations=2) - # show_mpl(img_bin_h | img_bin_v) + vizlogger.debug(img_bin_v, "tables04_isolate04_dilate_v.png") - # reduced filtersize from 100 to 80 to minimize splitting narrow cells - img_bin_h = apply_motion_blur(img_bin_h, 80, 0) - img_bin_v = apply_motion_blur(img_bin_v, 80, 90) + img_bin_h = apply_motion_blur(img_bin_h, 0) + vizlogger.debug(img_bin_h, "tables09_isolate05_blur_h.png") + img_bin_v = apply_motion_blur(img_bin_v, 90) + vizlogger.debug(img_bin_v, "tables10_isolate06_blur_v.png") img_bin_final = img_bin_h | img_bin_v - if show: - show_mpl(img_bin_final) - # changed threshold from 110 to 120 to minimize cell splitting - th1, img_bin_final = cv2.threshold(img_bin_final, 120, 255, cv2.THRESH_BINARY) - img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1) - # show_mpl(img_bin_final) + vizlogger.debug(img_bin_final, "tables11_isolate07_final.png") + + th1, img_bin_final = cv2.threshold(img_bin_final, 120, 255, cv2.THRESH_BINARY) + vizlogger.debug(img_bin_final, "tables10_isolate12_threshold.png") + img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1) + vizlogger.debug(img_bin_final, "tables11_isolate13_dilate.png") - # problem if layout parser detects too big of a layout box as in VV-748542.pdf p.22 img_bin_final = disconnect_non_existing_cells(img_bin_final, bounding_rects) - # show_mpl(img_bin_final) + vizlogger.debug(img_bin_final, "tables12_isolate14_disconnect.png") return img_bin_final def disconnect_non_existing_cells(img_bin, bounding_rects): - #TODO check if this even does anything for rect in bounding_rects: x, y, w, h = rect img_bin = cv2.rectangle(img_bin, (x, y), (x + w, y + h), (0, 0, 0), 5) return img_bin -# FIXME: does not work yet def has_table_shape(rects): assert isinstance(rects, list) @@ -111,48 +134,44 @@ def find_table_layout_boxes(image: np.array): def preprocess(image: np.array): - """ - - """ image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if len(image.shape) > 2 else image - th1, image = cv2.threshold(image, 195, 255, cv2.THRESH_BINARY) - image = ~image - return image + _, image = cv2.threshold(image, 195, 255, cv2.THRESH_BINARY) + return ~image def parse_table(image: np.array, show=False): - """ + """Runs the full table parsing process. + Args: + image (np.array): single PDF page, opened as PIL.Image object and converted to a numpy array + + Returns: + list: list of rectangles corresponding to table cells """ + def is_large_enough(stat): x1, y1, w, h, area = stat return area > 2000 and w > 35 and h > 25 image = preprocess(image) - if show: - show_mpl(image) - + table_layout_boxes = find_table_layout_boxes(image) image = isolate_vertical_and_horizontal_components(image, table_layout_boxes) - image = add_external_contours(image, image) - + _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) stats = np.vstack(list(filter(is_large_enough, stats))) rects = stats[:, :-1][2:] - # FIXME: produces false negatives for `data0/043d551b4c4c768b899eaece4466c836.pdf 1 --type table` - rects = remove_isolated(rects, input_sorted=True) - return list(rects) def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False): - """ - - """ + """ """ page = pdf2image.convert_from_path(pdf_path, first_page=page_index + 1, last_page=page_index + 1)[0] page = np.array(page) + if show: + show_mpl(page) if deskew: page, _ = deskew_histbased(page) @@ -161,5 +180,4 @@ def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False): if show: show_mpl(page) - else: - return page + vizlogger.debug(page, "tables15_final_output.png") diff --git a/cv_analysis/utils/display.py b/cv_analysis/utils/display.py index 7313650..4d346a5 100644 --- a/cv_analysis/utils/display.py +++ b/cv_analysis/utils/display.py @@ -9,6 +9,17 @@ def show_mpl(image): plt.show() +def save_mpl(image, path): + # fig, ax = plt.subplots(1, 1) + # figure = plt.gcf() + # figure.set_size_inches(16,12) + fig, ax = plt.subplots(1, 1) + fig.set_size_inches(20, 20) + ax.imshow(image, cmap="gray") + # plt.close() + plt.savefig(path) + + def show_cv2(image): cv2.imshow("", image) cv2.waitKey(0) diff --git a/cv_analysis/utils/visual_logging.py b/cv_analysis/utils/visual_logging.py new file mode 100644 index 0000000..19bbded --- /dev/null +++ b/cv_analysis/utils/visual_logging.py @@ -0,0 +1,22 @@ +import os +from cv_analysis.config import CONFIG +from cv_analysis.utils.display import save_mpl + +LEVEL = CONFIG.visual_logging.level +OUTPUT_FOLDER = CONFIG.visual_logging.output_folder + + +class VisualLogger: + def __init__(self): + self.level_is_debug = LEVEL == "DEBUG" + self.output_folder = OUTPUT_FOLDER + if not os.path.exists(self.output_folder): + os.mkdir(self.output_folder) + + def debug(self, img, name): + if self.level_is_debug: + output_path = os.path.join(self.output_folder, name) + save_mpl(img, output_path) + + +vizlogger = VisualLogger() diff --git a/scripts/annotate.py b/scripts/annotate.py index 03ab3db..a5d8e20 100644 --- a/scripts/annotate.py +++ b/scripts/annotate.py @@ -11,6 +11,7 @@ def parse_args(): parser.add_argument("pdf_path") parser.add_argument("page_index", type=int) parser.add_argument("--type", choices=["table", "redaction", "layout", "figure"]) + parser.add_argument("--show", action="store_true", default=False) args = parser.parse_args() @@ -19,11 +20,12 @@ def parse_args(): if __name__ == "__main__": args = parse_args() + #print(args.show) if args.type == "table": - annotate_tables_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + annotate_tables_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) elif args.type == "redaction": - annotate_redactions_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + annotate_redactions_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) elif args.type == "layout": - annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) elif args.type == "figure": - detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True) + detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show) diff --git a/scripts/client_mock.py b/scripts/client_mock.py index ddf8406..ffdd0ab 100644 --- a/scripts/client_mock.py +++ b/scripts/client_mock.py @@ -34,13 +34,7 @@ def parse_args(): def main(args): - # files = {"name": ( - # "name", - # open(args.pdf_path, "rb"), - # "file object corresponding to pdf file", - # {"operations": args.operations.split(",")} - # ) - # } + operations = args.operations.split(",") for operation in operations: print("****************************") diff --git a/src/run_service.py b/src/run_service.py index 68d1e08..269c2f4 100644 --- a/src/run_service.py +++ b/src/run_service.py @@ -20,6 +20,7 @@ from cv_analysis.config import CONFIG def suppress_user_warnings(): import warnings + warnings.filterwarnings("ignore")