From 4287f6d9878dd361489b8490eafd06f81df472ce Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Wed, 6 Jul 2022 16:56:12 +0200 Subject: [PATCH] removed debug prints --- pyinfra/file_descriptor_builder.py | 14 -------------- pyinfra/file_descriptor_manager.py | 6 +----- pyinfra/storage/adapters/s3.py | 1 - pyinfra/visitor/downloader.py | 3 --- test/fixtures/input.py | 9 ++------- test/fixtures/server.py | 7 ------- test/integration_tests/serve_test.py | 17 +++-------------- 7 files changed, 6 insertions(+), 51 deletions(-) diff --git a/pyinfra/file_descriptor_builder.py b/pyinfra/file_descriptor_builder.py index 8cf4da2..0dec1e9 100644 --- a/pyinfra/file_descriptor_builder.py +++ b/pyinfra/file_descriptor_builder.py @@ -39,21 +39,12 @@ class RedFileDescriptorBuilder(FileDescriptorBuilder): def build_file_descriptor(self, queue_item_body, end="input"): def pages(): - # return [queue_item_body["id"]] if end == "output" else [] if end == "input": - print("?!?!?!?!?!") if "id" in queue_item_body: return [queue_item_body["id"]] else: - # return queue_item_body["pages"] - # return [] if file_pattern["subdir"] else queue_item_body["pages"] - # return [] if len(queue_item_body["pages"]) > 1 else queue_item_body["pages"] - # return [] # single input - # return queue_item_body["pages"] # multi input - print(file_pattern) return queue_item_body["pages"] if file_pattern["multi"] else [] elif end == "output": - # return [queue_item_body["id"]] if "id" in queue_item_body else queue_item_body["pages"] return [queue_item_body["id"]] else: raise ValueError(f"Invalid argument: {end=}") # TODO: use an enum for `end` @@ -62,8 +53,6 @@ class RedFileDescriptorBuilder(FileDescriptorBuilder): file_pattern = self.operation2file_patterns[operation][end] - print(f"{end=}") - file_descriptor = { **project(queue_item_body, ["dossierId", "fileId", "pages"]), "pages": pages(), @@ -91,14 +80,11 @@ class RedFileDescriptorBuilder(FileDescriptorBuilder): def __build_page_regex(pages, subdir): n_pages = len(pages) - print("pages:", pages) - if n_pages > 1: page_re = "id:(" + "|".join(map(str, pages)) + ")." elif n_pages == 1: page_re = f"id:{pages[0]}." else: # no pages specified -> either all pages or no pages, depending on whether a subdir is specified - print("subdir", subdir) page_re = r"id:\d+." if subdir else "" return page_re diff --git a/pyinfra/file_descriptor_manager.py b/pyinfra/file_descriptor_manager.py index c69dfa9..13e715d 100644 --- a/pyinfra/file_descriptor_manager.py +++ b/pyinfra/file_descriptor_manager.py @@ -46,13 +46,9 @@ class FileDescriptorManager: return self.get_object_descriptor(queue_item_body, end="input") def get_output_object_descriptor(self, storage_upload_info): - print(1111111111111111111111111111111111111111111) - r = self.get_object_descriptor(storage_upload_info, end="output") - print(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", r) - return r + return self.get_object_descriptor(storage_upload_info, end="output") def get_object_descriptor(self, queue_item_body, end): - print(22222222222222222222222222222222222222222, end, queue_item_body) # TODO: this is complected with the Storage class API # FIXME: bad coupling return { diff --git a/pyinfra/storage/adapters/s3.py b/pyinfra/storage/adapters/s3.py index 15c17ce..cb06c67 100644 --- a/pyinfra/storage/adapters/s3.py +++ b/pyinfra/storage/adapters/s3.py @@ -27,7 +27,6 @@ class S3StorageAdapter(StorageAdapter): logger.debug(f"Uploading '{object_name}'...") data = io.BytesIO(data) self.__client.put_object(bucket_name, object_name, data, length=data.getbuffer().nbytes) - # input("uploaded") def get_object(self, bucket_name, object_name): logger.debug(f"Downloading '{object_name}'...") diff --git a/pyinfra/visitor/downloader.py b/pyinfra/visitor/downloader.py index 0324f1c..725b7c2 100644 --- a/pyinfra/visitor/downloader.py +++ b/pyinfra/visitor/downloader.py @@ -51,10 +51,7 @@ class Downloader: yield from self.storage.get_all_object_names(self.bucket_name, prefix=prefix) def get_pattern_filter(self, queue_item_body): - print("<<<<<<<<<<<<<<<<<<<<<<<<<<<", queue_item_body) file_pattern = self.file_descriptor_manager.build_input_matcher(queue_item_body) - print(f"{file_pattern=}") - # input() logger.debug(f"Filtering pattern: {file_pattern if len(file_pattern) <= 120 else (file_pattern[:120]+'...')}") matches_pattern = flift(file_pattern) return matches_pattern diff --git a/test/fixtures/input.py b/test/fixtures/input.py index 7fd7f68..f71ee5f 100644 --- a/test/fixtures/input.py +++ b/test/fixtures/input.py @@ -6,7 +6,6 @@ import pytest from PIL import Image from funcy import lmap, compose, flatten, lflatten, omit, second, first, lzip, merge -from pyinfra.server.debugging import inspect from pyinfra.server.dispatcher.dispatcher import Nothing from pyinfra.server.normalization import normalize_item from pyinfra.server.packing import pack, unpack @@ -81,7 +80,7 @@ def targets(data_message_pairs, input_data_items, operation, metadata, server_si if operation is Nothing: return Nothing - op = compose(lift(star(pack)), normalize_item, inspect("A"), operation) + op = compose(lift(star(pack)), normalize_item, operation) try: response_data, response_metadata = zip(*map(unpack, flatten(starmap(op, zip(input_data_items, metadata))))) @@ -148,10 +147,8 @@ def metadata(n_items, many_to_n): @pytest.fixture def queue_message_metadata(n_items, operation_name): - print() - print("n_items:", n_items) def metadata(i): - r = merge( + return merge( { "dossierId": "dossier_id", "fileId": f"file_id_{i}", @@ -159,8 +156,6 @@ def queue_message_metadata(n_items, operation_name): ({"operation": operation_name} if operation_name else {}), ({"pages": [0, 2, 3]} if n_items > 1 else {}), ) - print("metadata:", r) - return r return lmap(metadata, range(n_items)) diff --git a/test/fixtures/server.py b/test/fixtures/server.py index 4e78829..ede7ea1 100644 --- a/test/fixtures/server.py +++ b/test/fixtures/server.py @@ -54,7 +54,6 @@ def server(server_stream_function, buffer_size, operation_name): @pytest.fixture def operation_name(core_operation): operation_name = core_operation.__name__ - print("OPERATION NAME", operation_name) return operation_name @@ -108,20 +107,14 @@ def core_operation(many_to_n, item_type, one_to_many, analysis_task): yield upper(string, metadata), metadata def upper(string: bytes, metadata): - print("upper>>>>>>>>>>>", metadata) - print("???????????????????????????????", string) r = string.decode().upper().encode(), metadata - print("!!!!!!!!!!!!!!!!!!!!!", r) return r def extract(string: bytes, metadata): - print("extract>>>>>>>>>>>", metadata) - print(string.decode()) for i, c in project( dict(enumerate(string.decode())), metadata.get("pages", range(len(string.decode()))) ).items(): metadata["id"] = i - print(i) yield c.encode(), metadata def rotate(im: bytes, metadata): diff --git a/test/integration_tests/serve_test.py b/test/integration_tests/serve_test.py index c89553f..8fe1d15 100644 --- a/test/integration_tests/serve_test.py +++ b/test/integration_tests/serve_test.py @@ -82,11 +82,9 @@ from test.config import CONFIG as TEST_CONFIG False, ], ) -def test_serving(operation_name, server_process, bucket_name, components, targets, data_message_pairs, n_items, many_to_n): - - print(f"{operation_name=}") - print() - print() +def test_serving( + operation_name, server_process, bucket_name, components, targets, data_message_pairs, n_items, many_to_n +): storage, queue_manager, consumer, file_descriptor_manager = components @@ -112,12 +110,6 @@ def test_serving(operation_name, server_process, bucket_name, components, target # TODO: correctness of target should be validated as well, since production has become non-trivial assert sorted(outputs) == sorted(targets) - print(sorted(targets)) - print(sorted(outputs)) - - # sleep(2) - # print("---") - # input() @pytest.fixture @@ -157,9 +149,7 @@ def upload_data_to_folder_in_storage_and_publish_single_request_to_queue( for data, page in zip(map(first, data_message_pairs), pages): object_descriptor = file_descriptor_manager.get_input_object_descriptor({**ref_message, "id": page}) - print(f"{object_descriptor=}") object_descriptor["object_name"] = build_filepath(object_descriptor, page) - # input() storage.put_object(**object_descriptor, data=compress(data)) @@ -171,7 +161,6 @@ def build_filepath(object_descriptor, page): parts = object_name.split("/") path = "/".join(parts) path = re.sub(r"id:\d", f"id:{page}", path) - print(path) return path