From 186b4530f0fb8da85bdc60de79dd062c8fc00939 Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Tue, 31 Jan 2023 13:53:52 +0100 Subject: [PATCH] [WIP] Make texture show through page content --- cv_analysis/utils/conversion.py | 6 +- test/fixtures/page_generation/page.py | 187 +++++++++++++++++++++++--- 2 files changed, 171 insertions(+), 22 deletions(-) diff --git a/cv_analysis/utils/conversion.py b/cv_analysis/utils/conversion.py index c43f2c7..ce40d8b 100644 --- a/cv_analysis/utils/conversion.py +++ b/cv_analysis/utils/conversion.py @@ -40,10 +40,8 @@ class RectangleJSONEncoder(json.JSONEncoder): def normalize_image_format_to_array(image: Image_t): - return np.array(image) if isinstance(image, Image.Image) else image + return np.array(image).astype(np.uint8) if isinstance(image, Image.Image) else image def normalize_image_format_to_pil(image: Image_t): - if isinstance(image, np.ndarray): - return Image.fromarray(image) - return image + return Image.fromarray(image.astype(np.uint8)) if isinstance(image, np.ndarray) else image diff --git a/test/fixtures/page_generation/page.py b/test/fixtures/page_generation/page.py index 76958a9..81fba8e 100644 --- a/test/fixtures/page_generation/page.py +++ b/test/fixtures/page_generation/page.py @@ -3,7 +3,10 @@ import io import itertools import random import string +import sys import textwrap +from collections import Counter +from copy import deepcopy from enum import Enum from functools import lru_cache, partial from math import sqrt @@ -31,6 +34,9 @@ from cv_analysis.utils.merging import merge_related_rectangles from cv_analysis.utils.postprocessing import remove_overlapping, remove_included from cv_analysis.utils.spacial import area +logger.remove() +logger.add(sys.stderr, level="INFO") + random_seed = random.randint(0, 2**32 - 1) # random_seed = 3896311122 # random_seed = 1986343479 @@ -38,7 +44,7 @@ random_seed = random.randint(0, 2**32 - 1) # random_seed = 273244862 # empty large table # random_seed = 3717442900 # random_seed = 2508340737 -random_seed = 2212357755 +# random_seed = 2212357755 rnd = random.Random(random_seed) logger.info(f"Random seed: {random_seed}") @@ -180,6 +186,7 @@ Color = Tuple[int, int, int] def base_texture(request, size): texture = Image.open(TEST_PAGE_TEXTURES_DIR / (request.param + ".jpg")) texture = texture.resize(size) + texture.putalpha(255) return texture @@ -298,16 +305,22 @@ def overlay(images, mode=np.sum): @pytest.fixture -def texture(blank_page, base_texture): - texture = superimpose_texture_with_transparency(base_texture, blank_page) +def texture(tinted_blank_page, base_texture): + texture = superimpose_texture_with_transparency(base_texture, tinted_blank_page) return texture +@pytest.fixture +def tinted_blank_page(size, color, color_intensity): + tinted_page = Image.new("RGBA", size, color) + tinted_page.putalpha(color_intensity) + return tinted_page + + @pytest.fixture def blank_page(size, color, color_intensity): - color_image = Image.new("RGBA", size, color) - color_image.putalpha(color_intensity) - return color_image + page = Image.new("RGBA", size, color=(255, 255, 255, 0)) + return page def tint_image(src, color="#FFFFFF"): @@ -350,15 +363,40 @@ def size(dpi, orientation): return size -def superimpose_texture_with_transparency(page: Image, texture: Image, autocrop=True) -> Image: - """Superimposes a noise image with transparency onto a page image.""" +def superimpose_texture_with_transparency( + page: Image, + texture: Image, + crop_to_content=True, + pad=True, +) -> Image: + """Superimposes a noise image with transparency onto a page image. - if autocrop: + TODO: Rename page and texture to something more generic. + + Args: + page: The page image. + texture: The texture image. + crop_to_content: If True, the texture will be cropped to content (i.e. the bounding box of all non-transparent + parts of the texture image). + pad: If True, the texture will be padded to the size of the page. + + Returns: + Image where the texture is superimposed onto the page. + """ + page = normalize_image_format_to_pil(page) + texture = normalize_image_format_to_pil(texture) + + if crop_to_content: texture = texture.crop(texture.getbbox()) if page.size != texture.size: - logger.trace(f"Padding image before pasting to fit size {page.size}") - texture = pad_image_to_size(texture, page.size) + logger.trace(f"Size of page and texture do not match: {page.size} != {texture.size}") + if pad: + logger.trace(f"Padding texture before pasting to fit size {page.size}") + texture = pad_image_to_size(texture, page.size) + else: + logger.trace(f"Resizing texture before pasting to fit size {page.size}") + texture = texture.resize(page.size) assert page.size == texture.size assert texture.mode == "RGBA" @@ -375,7 +413,11 @@ def pad_image_to_size(image: Image, size: Tuple[int, int]) -> Image: if image.size[0] > size[0] or image.size[1] > size[1]: raise ValueError(f"Image size {image.size} is larger than target size {size}.") + # get average alpha value from image + # alpha = int(np.mean(np.array(image.split()[-1]))) or 255 + # padded = Image.new(image.mode, size, color=(255, 255, 255, alpha)) padded = Image.new(image.mode, size, color=255) + pasting_coords = compute_pasting_coordinates(image, padded) assert image.mode == "RGBA" padded.paste(image, pasting_coords) @@ -388,31 +430,134 @@ def compute_pasting_coordinates(smaller: Image, larger: Image.Image): @pytest.fixture -def page_with_content(texture, texture_fn) -> np.ndarray: +def page_with_content(blank_page, tinted_blank_page, texture, texture_fn) -> np.ndarray: """Creates a page with content""" - page = random_flip(texture) - page = texture_fn(page) page_partitioner = rnd.choice( [ TwoColumnPagePartitioner(), # RandomPagePartitioner(), ] ) - boxes = page_partitioner(page) + + ############################### + # texture = random_flip(texture) + # texture = texture_fn(texture) + # + # boxes = page_partitioner(texture) + # content_generator = ContentGenerator() + # boxes = content_generator(boxes) + # page = paste_contents(texture, boxes) + ############################### + + ################################ + boxes = page_partitioner(blank_page) content_generator = ContentGenerator() boxes = content_generator(boxes) - page = paste_contents(page, boxes) + page_content = paste_contents(blank_page, boxes) + + texture = random_flip(texture) + texture = texture_fn(texture) + + page_content = multiply_alpha_where_alpha_channel_is_nonzero(page_content, factor=0.6) + page = superimpose_texture_with_transparency(texture, page_content, crop_to_content=False) + ################################ + draw_boxes(page, boxes) - page = np.array(page) return page -def blend(a, b): +def provide_image_format(required_format): + def inner(fn): + def inner(image, *args, **kwargs): + + ret = fn(converter(image), *args, **kwargs) + + if get_image_format(image) != required_format: + ret = back_converter(ret) + + return ret + + converter = { + "array": normalize_image_format_to_array, + "pil": normalize_image_format_to_pil, + }[required_format] + + back_converter = { + "array": normalize_image_format_to_pil, + "pil": normalize_image_format_to_array, + }[required_format] + + return inner + + return inner + + +@provide_image_format("array") +def set_alpha_where_color_channels_are_nonzero(image: np.ndarray, alpha: int) -> np.ndarray: + """Sets the alpha channel of an image to a given value where the color channels are nonzero.""" + + assert image.ndim == 3 + assert image.shape[-1] == 4 + assert 0 <= alpha <= 255 + + image = image.copy() + image[..., -1] = np.where(np.logical_or.reduce(image[..., :-1] > 0, axis=-1), alpha, image[..., -1]) + return image + + +@provide_image_format("array") +def multiply_alpha_where_alpha_channel_is_nonzero(image: np.ndarray, factor: float) -> np.ndarray: + """Increases the alpha channel of an image where the alpha channel is nonzero.""" + + assert image.ndim == 3 + assert image.shape[-1] == 4 + + image = image.copy().astype(np.float32) + image[..., -1] = np.where(image[..., -1] > 0, image[..., -1] * factor, image[..., -1]) + image[..., -1] = np.clip(image[..., -1], 0, 255) + + assert image.max() <= 255 + assert image.min() >= 0 + return image + + +@provide_image_format("array") +def set_alpha_where_alpha_channel_is_nonzero(image: np.ndarray, alpha: int) -> np.ndarray: + """Sets the alpha channel of an image to a given value where the alpha channel is nonzero.""" + + assert image.ndim == 3 + assert image.shape[-1] == 4 + assert 0 <= alpha <= 255 + + image = image.copy() + image[..., -1] = np.where(image[..., -1] > 0, alpha, image[..., -1]) + return image + + +def get_image_format(image): + if isinstance(image, np.ndarray): + return "array" + elif isinstance(image, Image.Image): + return "pil" + else: + raise ValueError(f"Unknown image format: {type(image)}") + + +def blend(a: np.ndarray, b: np.ndarray): """Reference: https://stackoverflow.com/a/52143032""" + + assert a.max() <= 255 + assert a.min() >= 0 + + assert b.max() <= 255 + assert b.min() >= 0 + a = a.astype(float) / 255 b = b.astype(float) / 255 # make float on range 0-1 + print(a.shape, b.shape) + mask = a >= 0.5 # generate boolean mask of everywhere a > 0.5 ab = np.zeros_like(a) # generate an output container for the blended image @@ -420,6 +565,11 @@ def blend(a, b): ab[~mask] = (2 * a * b)[~mask] # 2ab everywhere a<0.5 ab[mask] = (1 - 2 * (1 - a) * (1 - b))[mask] # else this + assert ab.max() <= 1 + assert ab.min() >= 0 + + return (ab * 255).astype(np.uint8) + class ContentRectangle(Rectangle): def __init__(self, x1, y1, x2, y2, content=None): @@ -1425,6 +1575,7 @@ def paste_content(page, content_box: ContentRectangle): def paste_contents(page, contents: Iterable[ContentRectangle]): + page = deepcopy(page) for content in contents: paste_content(page, content) return page