diff --git a/cv_analysis/server/pipeline.py b/cv_analysis/server/pipeline.py index fefb3ce..bca7f0c 100644 --- a/cv_analysis/server/pipeline.py +++ b/cv_analysis/server/pipeline.py @@ -38,14 +38,14 @@ def make_analysis_pipeline(analysis_fn, formatter, dpi): return analyse_pipeline -def table_parsing_formatter(rects, page, dpi): +def table_parsing_formatter(rects, page: ImagePlus, dpi): def format_rect(rect: Rectangle): rect_plus = RectanglePlus.from_pixels(*rect.xyxy(), page.info, alpha=False, dpi=dpi) return rect_plus.asdict(derotate=True) bboxes = lmap(format_rect, rects) - return {"pageInfo": page.asdict(), "tableCells": bboxes} + return {"pageInfo": page.asdict(natural_index=True), "tableCells": bboxes} def figure_detection_formatter(rects, page, dpi): diff --git a/cv_analysis/table_parsing.py b/cv_analysis/table_parsing.py index bb3105f..06b15a8 100644 --- a/cv_analysis/table_parsing.py +++ b/cv_analysis/table_parsing.py @@ -1,6 +1,6 @@ import cv2 import numpy as np -from funcy import lmap +from funcy import lmap, lfilter from cv_analysis.layout_parsing import parse_layout from cv_analysis.utils.postprocessing import remove_isolated # xywh_to_vecs, xywh_to_vec_rect, adjacent1d @@ -106,8 +106,11 @@ def turn_connected_components_into_rects(image: np.array): _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S) - stats = np.vstack(list(filter(is_large_enough, stats))) - return stats[:, :-1][2:] + stats = lfilter(is_large_enough, stats) + if stats: + stats = np.vstack(stats) + return stats[:, :-1][2:] + return [] def parse_tables(image: np.array, show=False): diff --git a/incl/pdf2image b/incl/pdf2image index 9bb5a86..f7292c3 160000 --- a/incl/pdf2image +++ b/incl/pdf2image @@ -1 +1 @@ -Subproject commit 9bb5a86310f065b852e16679cf37d5c939c0cacd +Subproject commit f7292c30ad7c7ae5f07cee6925adda096301b60a diff --git a/incl/pyinfra b/incl/pyinfra index 88b4c5c..94beb54 160000 --- a/incl/pyinfra +++ b/incl/pyinfra @@ -1 +1 @@ -Subproject commit 88b4c5c7ce9852b8aa4bdd6b760f4c8b708df62b +Subproject commit 94beb544fac425257ab2ba9e9ad4ad53abc32c71 diff --git a/scripts/save_figure_detection_data.py b/scripts/run_analysis_pipeline.py similarity index 59% rename from scripts/save_figure_detection_data.py rename to scripts/run_analysis_pipeline.py index d25e624..3c8e37f 100644 --- a/scripts/save_figure_detection_data.py +++ b/scripts/run_analysis_pipeline.py @@ -8,21 +8,22 @@ from cv_analysis.server.pipeline import get_analysis_pipeline def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("pdf") + parser.add_argument("--type", "-t", choices=["table", "layout", "figure"], required=True) return parser.parse_args() if __name__ == "__main__": args = parse_args() - detect_figures = get_analysis_pipeline("figure") + analysis_fn = get_analysis_pipeline(args.type) with open(args.pdf, "rb") as f: pdf_bytes = f.read() - results = list(detect_figures(pdf_bytes)) + results = list(analysis_fn(pdf_bytes)) folder = Path(args.pdf).parent file_stem = Path(args.pdf).stem - with open(f"{folder}/{file_stem}_figures.json", "w+") as f: - json.dump(results, f, indent=2) \ No newline at end of file + with open(f"{folder}/{file_stem}_{args.type}.json", "w+") as f: + json.dump(results, f, indent=2) diff --git a/test/unit_tests/server_pipeline_test.py b/test/unit_tests/server_pipeline_test.py index 181f855..a5e809b 100644 --- a/test/unit_tests/server_pipeline_test.py +++ b/test/unit_tests/server_pipeline_test.py @@ -24,8 +24,17 @@ def expected_formatted_analysis_result(operation): if operation == "table": return [ { - "pageInfo": {"number": 0, "rotation": 0, "width": 595.0, "height": 842.0}, - "tableCells": [{"x0": 0.0, "y0": 0.0, "x1": 15.12, "y1": 15.12, "width": 15.12, "height": 15.12}], + "pageInfo": {"number": 1, "rotation": 0, "width": 595.0, "height": 842.0}, + "tableCells": [ + { + "x0": 0.0, + "y0": 826.8800048828125, + "x1": 15.119999885559082, + "y1": 842.0, + "width": 15.119999885559082, + "height": 15.1199951171875, + } + ], } ] if operation == "figure":