Marmelator/cv-analysis-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

added visual logger for development

Browse Source
This commit is contained in:
Isaac Riley 2022-04-21 15:10:35 +02:00
parent 0ea556a7e0
commit 88bb8dbddf
10 changed files with 133 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.yaml
View File

@ -20,4 +20,8 @@ deskew:
verbose: False
filter_strength_h: 3
test_dummy: test_dummy
test_dummy: test_dummy
visual_logging:
level: $LOGGING_LEVEL_ROOT|DEBUG
output_folder: /tmp/debug/

8
cv_analysis/figure_detection.py
View File

@ -8,6 +8,7 @@ from cv_analysis.utils.draw import draw_rectangles
from cv_analysis.utils.post_processing import remove_included
from cv_analysis.utils.filters import is_large_enough, has_acceptable_format
from cv_analysis.utils.text import remove_primary_text_regions
from cv_analysis.utils.visual_logging import vizlogger
def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6):
@ -17,8 +18,10 @@ def is_likely_figure(cont, min_area=5000, max_width_to_hight_ratio=6):
def detect_figures(image: np.array):
image = image.copy()
vizlogger.debug(image, "figures01_start.png")
image = remove_primary_text_regions(image)
vizlogger.debug(image, "figures02_remove_text.png")
cnts = detect_large_coherent_structures(image)
cnts = filter(is_likely_figure, cnts)
@ -35,8 +38,7 @@ def detect_figures_in_pdf(pdf_path, page_index=1, show=False):
redaction_contours = detect_figures(page)
page = draw_rectangles(page, redaction_contours)
vizlogger.debug(page, "figures03_final.png")
if show:
show_mpl(page)
else:
return page

19
cv_analysis/layout_parsing.py
View File

@ -9,6 +9,7 @@ from pdf2image import pdf2image
from cv_analysis.utils.display import show_mpl
from cv_analysis.utils.draw import draw_rectangles
from cv_analysis.utils.post_processing import remove_overlapping, remove_included, has_no_parent
from cv_analysis.utils.visual_logging import vizlogger
def is_likely_segment(rect, min_area=100):
@ -35,11 +36,17 @@ def parse_layout(image: np.array):
if len(image_.shape) > 2:
image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2GRAY)
vizlogger.debug(image_, "layout01_start.png")
image_ = cv2.GaussianBlur(image_, (7, 7), 0)
vizlogger.debug(image_, "layout02_blur.png")
thresh = cv2.threshold(image_, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
vizlogger.debug(image_, "layout03_theshold.png")
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
vizlogger.debug(kernel, "layout04_kernel.png")
dilate = cv2.dilate(thresh, kernel, iterations=4)
vizlogger.debug(dilate, "layout05_dilate.png")
rects = list(find_segments(dilate))
@ -48,12 +55,17 @@ def parse_layout(image: np.array):
x, y, w, h = rect
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 0), -1)
cv2.rectangle(image, (x, y), (x + w, y + h), (255, 255, 255), 7)
vizlogger.debug(image, "layout06_rectangles.png")
_, image = cv2.threshold(image, 254, 255, cv2.THRESH_BINARY)
vizlogger.debug(image, "layout07_threshold.png")
image = ~image
vizlogger.debug(image, "layout08_inverse.png")
if len(image.shape) > 2:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
vizlogger.debug(image, "layout09_convertcolor.png")
rects = find_segments(image)
# <- End of meta detection
@ -70,12 +82,11 @@ def annotate_layout_in_pdf(pdf_path, page_index=1, show=False):
rects = parse_layout(page)
page = draw_rectangles(page, rects)
vizlogger.debug(page, "layout10_output.png")
if show:
show_mpl(page)
else:
return page
"""
def find_layout_boxes(image: np.array):
@ -125,4 +136,4 @@ def annotate_layout_in_pdf(pdf_path, page_index=1):
ax.imshow(page)
plt.show()
"""
"""

10
cv_analysis/redaction_detection.py
View File

@ -8,6 +8,7 @@ from iteration_utilities import starfilter, first
from cv_analysis.utils.display import show_mpl
from cv_analysis.utils.draw import draw_contours
from cv_analysis.utils.filters import is_large_enough, is_filled, is_boxy
from cv_analysis.utils.visual_logging import vizlogger
def is_likely_redaction(contour, hierarchy, min_area):
@ -15,15 +16,18 @@ def is_likely_redaction(contour, hierarchy, min_area):
def find_redactions(image: np.array, min_normalized_area=200000):
vizlogger.debug(image, "redactions01_start.png")
min_normalized_area /= 200 # Assumes 200 DPI PDF -> image conversion resolution
if len(image.shape) > 2:
gray = ~cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
else:
gray = ~image
vizlogger.debug(gray, "redactions02_gray.png")
blurred = cv2.GaussianBlur(gray, (5, 5), 1)
vizlogger.debug(blurred, "redactions03_blur.png")
thresh = cv2.threshold(blurred, 252, 255, cv2.THRESH_BINARY)[1]
vizlogger.debug(blurred, "redactions04_threshold.png")
contours, hierarchies = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
@ -43,8 +47,8 @@ def annotate_redactions_in_pdf(pdf_path, page_index=1, show=False):
redaction_contours = find_redactions(page)
page = draw_contours(page, redaction_contours)
vizlogger.debug(page, "redactions05_output.png")
if show:
show_mpl(page)
else:
return page

102
cv_analysis/table_parsing.py
View File

@ -2,81 +2,104 @@ from functools import partial
from itertools import chain, starmap
from operator import attrgetter
from os.path import join
import cv2
import numpy as np
from pdf2image import pdf2image
from cv_analysis.utils.display import show_mpl
from cv_analysis.utils.draw import draw_rectangles
from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d, remove_isolated
from cv_analysis.utils.post_processing import xywh_to_vecs, xywh_to_vec_rect, adjacent1d
from cv_analysis.utils.deskew import deskew_histbased
from cv_analysis.utils.visual_logging import vizlogger
from cv_analysis.layout_parsing import parse_layout
def add_external_contours(image, img):
contours, _ = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
# contours = filter(partial(is_large_enough, min_area=5000000), contours)
for cnt in contours:
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(image, (x, y), (x + w, y + h), 255, 1)
vizlogger.debug(image, "external_contours.png")
return image
def apply_motion_blur(image, size, angle):
"""
def apply_motion_blur(image: np.array, angle, size=80):
"""Solidifies and slightly extends detected lines.
Args:
image (np.array): page image as array
angle: direction in which to apply blur, 0 or 90
size (int): kernel size; 80 found empirically to work well
Returns:
np.array
"""
k = np.zeros((size, size), dtype=np.float32)
vizlogger.debug(k, "tables08_blur_kernel1.png")
k[(size - 1) // 2, :] = np.ones(size, dtype=np.float32)
vizlogger.debug(k, "tables09_blur_kernel2.png")
k = cv2.warpAffine(k, cv2.getRotationMatrix2D((size / 2 - 0.5, size / 2 - 0.5), angle, 1.0), (size, size))
vizlogger.debug(k, "tables10_blur_kernel3.png")
k = k * (1.0 / np.sum(k))
return cv2.filter2D(image, -1, k)
vizlogger.debug(k, "tables11_blur_kernel4.png")
blurred = cv2.filter2D(image, -1, k)
return blurred
def isolate_vertical_and_horizontal_components(img_bin, bounding_rects, show=False):
def isolate_vertical_and_horizontal_components(img_bin, bounding_rects):
"""Identifies and reinforces horizontal and vertical lines in a binary image.
Args:
img_bin (np.array): array corresponding to single binarized page image
bounding_rects (list): list of layout boxes of the form (x, y, w, h), potentially containing tables
Returns:
np.array
"""
line_min_width = 48
kernel_h = np.ones((1, line_min_width), np.uint8)
kernel_v = np.ones((line_min_width, 1), np.uint8)
img_bin_h = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_h)
vizlogger.debug(img_bin_h, "tables01_isolate01_img_bin_h.png")
img_bin_v = cv2.morphologyEx(img_bin, cv2.MORPH_OPEN, kernel_v)
if show:
show_mpl(img_bin_h | img_bin_v)
vizlogger.debug(img_bin_v, "tables02_isolate02_img_bin_v.png")
kernel_h = np.ones((1, 30), np.uint8)
kernel_v = np.ones((30, 1), np.uint8)
img_bin_h = cv2.dilate(img_bin_h, kernel_h, iterations=2)
vizlogger.debug(img_bin_h, "tables03_isolate03_dilate_h.png")
img_bin_v = cv2.dilate(img_bin_v, kernel_v, iterations=2)
# show_mpl(img_bin_h | img_bin_v)
vizlogger.debug(img_bin_v, "tables04_isolate04_dilate_v.png")
# reduced filtersize from 100 to 80 to minimize splitting narrow cells
img_bin_h = apply_motion_blur(img_bin_h, 80, 0)
img_bin_v = apply_motion_blur(img_bin_v, 80, 90)
img_bin_h = apply_motion_blur(img_bin_h, 0)
vizlogger.debug(img_bin_h, "tables09_isolate05_blur_h.png")
img_bin_v = apply_motion_blur(img_bin_v, 90)
vizlogger.debug(img_bin_v, "tables10_isolate06_blur_v.png")
img_bin_final = img_bin_h | img_bin_v
if show:
show_mpl(img_bin_final)
# changed threshold from 110 to 120 to minimize cell splitting
th1, img_bin_final = cv2.threshold(img_bin_final, 120, 255, cv2.THRESH_BINARY)
img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1)
# show_mpl(img_bin_final)
vizlogger.debug(img_bin_final, "tables11_isolate07_final.png")
th1, img_bin_final = cv2.threshold(img_bin_final, 120, 255, cv2.THRESH_BINARY)
vizlogger.debug(img_bin_final, "tables10_isolate12_threshold.png")
img_bin_final = cv2.dilate(img_bin_final, np.ones((1, 1), np.uint8), iterations=1)
vizlogger.debug(img_bin_final, "tables11_isolate13_dilate.png")
# problem if layout parser detects too big of a layout box as in VV-748542.pdf p.22
img_bin_final = disconnect_non_existing_cells(img_bin_final, bounding_rects)
# show_mpl(img_bin_final)
vizlogger.debug(img_bin_final, "tables12_isolate14_disconnect.png")
return img_bin_final
def disconnect_non_existing_cells(img_bin, bounding_rects):
#TODO check if this even does anything
for rect in bounding_rects:
x, y, w, h = rect
img_bin = cv2.rectangle(img_bin, (x, y), (x + w, y + h), (0, 0, 0), 5)
return img_bin
# FIXME: does not work yet
def has_table_shape(rects):
assert isinstance(rects, list)
@ -111,48 +134,44 @@ def find_table_layout_boxes(image: np.array):
def preprocess(image: np.array):
"""
"""
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if len(image.shape) > 2 else image
th1, image = cv2.threshold(image, 195, 255, cv2.THRESH_BINARY)
image = ~image
return image
_, image = cv2.threshold(image, 195, 255, cv2.THRESH_BINARY)
return ~image
def parse_table(image: np.array, show=False):
"""
"""Runs the full table parsing process.
Args:
image (np.array): single PDF page, opened as PIL.Image object and converted to a numpy array
Returns:
list: list of rectangles corresponding to table cells
"""
def is_large_enough(stat):
x1, y1, w, h, area = stat
return area > 2000 and w > 35 and h > 25
image = preprocess(image)
if show:
show_mpl(image)
table_layout_boxes = find_table_layout_boxes(image)
image = isolate_vertical_and_horizontal_components(image, table_layout_boxes)
image = add_external_contours(image, image)
_, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S)
stats = np.vstack(list(filter(is_large_enough, stats)))
rects = stats[:, :-1][2:]
# FIXME: produces false negatives for `data0/043d551b4c4c768b899eaece4466c836.pdf 1 --type table`
rects = remove_isolated(rects, input_sorted=True)
return list(rects)
def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False):
"""
"""
""" """
page = pdf2image.convert_from_path(pdf_path, first_page=page_index + 1, last_page=page_index + 1)[0]
page = np.array(page)
if show:
show_mpl(page)
if deskew:
page, _ = deskew_histbased(page)
@ -161,5 +180,4 @@ def annotate_tables_in_pdf(pdf_path, page_index=0, deskew=False, show=False):
if show:
show_mpl(page)
else:
return page
vizlogger.debug(page, "tables15_final_output.png")

11
cv_analysis/utils/display.py
View File

@ -9,6 +9,17 @@ def show_mpl(image):
plt.show()
def save_mpl(image, path):
# fig, ax = plt.subplots(1, 1)
# figure = plt.gcf()
# figure.set_size_inches(16,12)
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(20, 20)
ax.imshow(image, cmap="gray")
# plt.close()
plt.savefig(path)
def show_cv2(image):
cv2.imshow("", image)
cv2.waitKey(0)

22
cv_analysis/utils/visual_logging.py Normal file
View File

@ -0,0 +1,22 @@
import os
from cv_analysis.config import CONFIG
from cv_analysis.utils.display import save_mpl
LEVEL = CONFIG.visual_logging.level
OUTPUT_FOLDER = CONFIG.visual_logging.output_folder
class VisualLogger:
def __init__(self):
self.level_is_debug = LEVEL == "DEBUG"
self.output_folder = OUTPUT_FOLDER
if not os.path.exists(self.output_folder):
os.mkdir(self.output_folder)
def debug(self, img, name):
if self.level_is_debug:
output_path = os.path.join(self.output_folder, name)
save_mpl(img, output_path)
vizlogger = VisualLogger()

10
scripts/annotate.py
View File

@ -11,6 +11,7 @@ def parse_args():
parser.add_argument("pdf_path")
parser.add_argument("page_index", type=int)
parser.add_argument("--type", choices=["table", "redaction", "layout", "figure"])
parser.add_argument("--show", action="store_true", default=False)
args = parser.parse_args()
@ -19,11 +20,12 @@ def parse_args():
if __name__ == "__main__":
args = parse_args()
#print(args.show)
if args.type == "table":
annotate_tables_in_pdf(args.pdf_path, page_index=args.page_index, show=True)
annotate_tables_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show)
elif args.type == "redaction":
annotate_redactions_in_pdf(args.pdf_path, page_index=args.page_index, show=True)
annotate_redactions_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show)
elif args.type == "layout":
annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=True)
annotate_layout_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show)
elif args.type == "figure":
detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=True)
detect_figures_in_pdf(args.pdf_path, page_index=args.page_index, show=args.show)

8
scripts/client_mock.py
View File

@ -34,13 +34,7 @@ def parse_args():
def main(args):
# files = {"name": (
# "name",
# open(args.pdf_path, "rb"),
# "file object corresponding to pdf file",
# {"operations": args.operations.split(",")}
# )
# }
operations = args.operations.split(",")
for operation in operations:
print("****************************")

1
src/run_service.py
View File

@ -20,6 +20,7 @@ from cv_analysis.config import CONFIG
def suppress_user_warnings():
import warnings
warnings.filterwarnings("ignore")