merge master

cv_analysis/layout_parsing.py

@@ -7,7 +7,10 @@ import cv2
 import numpy as np
 
 
-from cv_analysis.utils.connect_rects import connect_related_rects2
+from cv_analysis.utils.connect_rects import connect_related_rects
+from cv_analysis.utils.display import show_image_mpl
+from cv_analysis.utils.draw import draw_rectangles
+from cv_analysis.utils.label_rects import label_rects
 from cv_analysis.utils.structures import Rectangle
 from cv_analysis.utils.postprocessing import (
     remove_overlapping,
@@ -53,7 +56,7 @@ def fill_in_component_area(image, rect):
 def parse_layout(image: np.array):
     image = image.copy()
     image_ = image.copy()
-
+    #show_image_mpl(image)
     if len(image_.shape) > 2:
         image_ = cv2.cvtColor(image_, cv2.COLOR_BGR2GRAY)
 
@@ -80,8 +83,7 @@ def parse_layout(image: np.array):
     rects = remove_included(rects)
 
     rects = map(lambda r: r.xywh(), rects)
-    rects = connect_related_rects2(rects)
+    rects = connect_related_rects(rects)
     rects = list(map(Rectangle.from_xywh, rects))
-    rects = remove_included(rects)
-
+    # rects = remove_included(rects)
     return rects

cv_analysis/table_parsing.py

@@ -6,7 +6,6 @@ import cv2
 import numpy as np
 from funcy import lmap
 
-from cv_analysis.layout_parsing import parse_layout
 from cv_analysis.utils.postprocessing import remove_isolated  # xywh_to_vecs, xywh_to_vec_rect, adjacent1d
 from cv_analysis.utils.structures import Rectangle
 from cv_analysis.utils.visual_logging import vizlogger
@@ -86,32 +85,36 @@ def isolate_vertical_and_horizontal_components(img_bin):
     return img_bin_final
 
 
-def find_table_layout_boxes(image: np.array):
-    def is_large_enough(box):
-        (x, y, w, h) = box
-        if w * h >= 100000:
-            return Rectangle.from_xywh(box)
-
-    layout_boxes = parse_layout(image)
-    a = lmap(is_large_enough, layout_boxes)
-    return lmap(is_large_enough, layout_boxes)
-
-
 def preprocess(image: np.array):
     image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) if len(image.shape) > 2 else image
     _, image = cv2.threshold(image, 195, 255, cv2.THRESH_BINARY)
     return ~image
 
 
+# def turn_connected_components_into_rects(image: np.array):
+#     def is_large_enough(stat):
+#         x1, y1, w, h, area = stat
+#         return area > 2000 and w > 35 and h > 25
+#
+#     _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S)
+#
+#     stats = np.vstack(list(filter(is_large_enough, stats)))
+#     rects = list(map(Rectangle.from_xywh, stats[:, :-1][2:]))
+#     return remove_isolated(rects)
+
+
 def turn_connected_components_into_rects(image: np.array):
     def is_large_enough(stat):
         x1, y1, w, h, area = stat
         return area > 2000 and w > 35 and h > 25
 
     _, _, stats, _ = cv2.connectedComponentsWithStats(~image, connectivity=8, ltype=cv2.CV_32S)
-
+    try:
         stats = np.vstack(list(filter(is_large_enough, stats)))
-    return stats[:, :-1][2:]
+        rects = list(map(Rectangle.from_xywh, stats[:, :-1][2:]))
+        return remove_isolated(rects)
+    except ValueError:
+        return []
 
 
 def parse_tables(image: np.array, show=False):
@@ -128,9 +131,9 @@ def parse_tables(image: np.array, show=False):
     image = isolate_vertical_and_horizontal_components(image)
     rects = turn_connected_components_into_rects(image)
     #print(rects, "\n\n")
-    rects = list(map(Rectangle.from_xywh, rects))
+    #rects = list(map(Rectangle.from_xywh, rects))
     #print(rects, "\n\n")
-    rects = remove_isolated(rects)
+    #rects = remove_isolated(rects)
     #print(rects, "\n\n")
     
     return rects

cv_analysis/utils/connect_rects.py

@@ -65,38 +65,8 @@ def rects_not_the_same(r):
     return r[0] != r[1]
 
 
-def find_related_rects(rects):
-    rect_pairs = list(filter(is_related, combinations(rects, 2)))
-    rect_pairs = list(filter(rects_not_the_same, rect_pairs))
-    if not rect_pairs:
-        return [], rects
-    rel_rects = list(set([rect for pair in rect_pairs for rect in pair]))
-    unrel_rects = [rect for rect in rects if rect not in rel_rects]
-    return rect_pairs, unrel_rects
 
-
-def connect_related_rects(rects):
-    rects_to_connect, rects_new = find_related_rects(rects)
-
-    while len(rects_to_connect) > 0:
-        rects_fused = list(starmap(fuse_rects, rects_to_connect))
-        rects_fused = list(dict.fromkeys(rects_fused))
-
-        if len(rects_fused) == 1:
-            rects_new += rects_fused
-            rects_fused = []
-
-        rects_to_connect, connected_rects = find_related_rects(rects_fused)
-        rects_new += connected_rects
-
-        if len(rects_to_connect) > 1 and len(set(rects_to_connect)) == 1:
-            rects_new.append(rects_fused[0])
-            rects_to_connect = []
-
-    return rects_new
-
-
-def connect_related_rects2(rects: Iterable[tuple]):
+def connect_related_rects(rects: Iterable[tuple]):
     rects = list(rects)
     current_idx = 0
 

cv_analysis/utils/label_rects.py

@@ -0,0 +1,93 @@
+from itertools import starmap
+from typing import Iterable
+
+import cv2
+import numpy as np
+
+from cv_analysis.figure_detection.text import remove_primary_text_regions, apply_threshold_to_image
+from cv_analysis.table_parsing import preprocess, isolate_vertical_and_horizontal_components, \
+    turn_connected_components_into_rects
+from cv_analysis.utils.display import show_image_mpl
+
+
+def area_is_bigger_than(rect: tuple, maxarea=100000):
+    x, y, w, h = rect
+    return w * h >= maxarea
+
+
+def define_rect(rect_img, original_position):
+    # print(original_position)
+    # show_image_mpl(rect_img)
+    xo, yo, wo, ho = original_position
+    rect_img_inv = preprocess(rect_img)
+    # print("pixel density inverted img", pixel_density(rect_img_inv))
+    grid_inv = isolate_vertical_and_horizontal_components(rect_img_inv)
+    cnts, _ = cv2.findContours(image=grid_inv, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE)
+    if cnts:
+        rects = turn_connected_components_into_rects(grid_inv)
+        rects = map(lambda r: r.xywh(), rects)
+        bbox = list((cv2.boundingRect(c) for c in cnts))
+        if len(list(rects)) > 1 and len(bbox) == 1:
+            x, y, w, h = bbox[0]
+            w_img, h_img = rect_img.shape
+            if w * h / (w_img * h_img) >= 0.75:
+                # print("is table")
+                return "table"
+            else:
+                # show_image_mpl(rect_img)
+                # print(" table detected but to small for layout rect, so cant be table, maybe figure?")
+                return "other"
+
+        else:
+            if is_header(yo + ho):
+                # print("is header component")
+                return "header component"
+            elif is_footer(yo):
+                # print("is footer component")
+                return "footer component"
+            else:
+                # print("single cell or no connected components, maybe figure?")
+                return "other"
+
+    else:
+        if is_header(yo + ho):
+            # print("is header text")
+            return "header text"
+        elif is_footer(yo):
+            # print("is footer text")
+            return "footer text"
+        else:
+            # print("is text")
+            return "text"
+
+
+def is_header(y):
+    return y < 200
+
+
+def is_footer(y):
+    return y > 2100
+
+
+def is_text(img):
+    show_image_mpl(img)
+    cleaned = remove_primary_text_regions(img)
+    show_image_mpl(cleaned)
+    return pixel_density(cleaned) < 0.05
+
+
+def pixel_density(img):
+    pixels = np.count_nonzero(img)
+    density = pixels / img.size
+    return density
+
+
+
+def label_rects(image: np.array, rects: Iterable[tuple]):
+    def crop_image_rects(rect):
+        x, y, w, h = rect
+        return image[y:y + h, x:x + w]
+
+    rect_images = map(crop_image_rects, rects)
+    rect_labels = starmap(define_rect, zip(rect_images, rects))
+    return rect_labels

cv_analysis/utils/sort_rects.py

@@ -0,0 +1,96 @@
+from itertools import starmap
+from typing import Iterable
+
+import cv2
+import numpy as np
+
+from cv_analysis.figure_detection.text import remove_primary_text_regions, apply_threshold_to_image
+from cv_analysis.table_parsing import preprocess, isolate_vertical_and_horizontal_components, \
+    turn_connected_components_into_rects
+from cv_analysis.utils.display import show_image_mpl
+
+
+def area_is_bigger_than(rect: tuple, maxarea=100000):
+    x, y, w, h = rect
+    return w * h >= maxarea
+
+
+def define_rect(rect_img, original_position):
+    show_image_mpl(rect_img)
+    x,y,w,h = original_position
+    if is_header(y+h):
+        print(original_position, " is header")
+        return "header"
+    elif is_footer(y):
+        print(original_position, " is footer")
+        return "footer"
+    elif is_table(rect_img):
+        print(original_position, " is table")
+        return "table"
+    elif is_text(rect_img):
+        print(original_position, " is text")
+        return "text"
+    else:
+        return "other"
+
+
+def is_table(rect_img):
+    rect_img_inv = preprocess(rect_img)
+    grid_inv = isolate_vertical_and_horizontal_components(rect_img_inv)
+    cnts, _ = cv2.findContours(image=grid_inv, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE)
+    if cnts:
+        rects = turn_connected_components_into_rects(grid_inv)
+        rects = map(lambda r: r.xywh(), rects)
+        bbox = list((cv2.boundingRect(c) for c in cnts))
+        if len(list(rects)) > 1 and len(bbox) == 1:
+            x, y, w, h = bbox[0]
+            w_img, h_img = rect_img.shape
+            if w * h / (w_img * h_img) >= 0.75:
+                #print("is table")
+                return True
+            else:
+                print(" table detected but to small for layout rect, so cant be table, maybe figure?")
+                return False
+
+        else:
+            print("single cell or no connected components, maybe figure?")
+            return False
+
+    else:
+        print("not a table, but text?")
+        return False
+
+def is_header(y):
+    return y < 200
+
+
+def is_footer(y):
+    return y > 2150
+
+
+def is_text(img):
+    show_image_mpl(img)
+    cleaned = remove_primary_text_regions(img)
+    show_image_mpl(cleaned)
+    return pixel_density(cleaned) < 0.05
+
+
+def pixel_density(img):
+    pixels = np.count_nonzero(img)
+    density = pixels / img.size
+    return density
+
+
+def annotate_rect(rect, rect_img):
+    pass
+
+
+def label_rects(rects: Iterable[tuple], image: np.array):
+    def crop_image_rects(rect):
+        x, y, w, h = rect
+        return image[y:y + h, x:x + w]
+
+    rect_images = map(crop_image_rects, rects)
+    rect_labels = starmap(define_rect, zip(rect_images, rects))
+    print(rect_labels)
+    return rect_labels

incl/pyinfra

@@ -1 +1 @@
-Subproject commit 88b4c5c7ce9852b8aa4bdd6b760f4c8b708df62b
+Subproject commit 71ad2af4eb278a3718ad5385b06f07faa9059e9f

scripts/annotate_pdf.py

@@ -13,6 +13,8 @@ from cv_analysis.table_parsing import parse_tables
 from cv_analysis.utils.draw import draw_rectangles
 from pdf2img.conversion import convert_pages_to_images
 
+from cv_analysis.utils.sort_rects import label_rects
+
 
 def parse_args():
     parser = argparse.ArgumentParser()