import gzip import json import logging from itertools import starmap, repeat, chain from operator import itemgetter import pytest from frozendict import frozendict from funcy import lfilter, compose, lzip from pyinfra.default_objects import ( get_callback, get_response_strategy, ) from pyinfra.queue.consumer import Consumer from pyinfra.server.packing import bytes_to_string, unpack, pack from pyinfra.utils.func import star from pyinfra.visitor import get_object_descriptor, QueueVisitor from test.utils.input import adorn_data_with_storage_info logger = logging.getLogger(__name__) def freeze(data, metadata): return data, frozendict(metadata) @pytest.fixture def components(endpoint, bucket_name, queue_manager, storage): callback = get_callback(endpoint) consumer = Consumer(callback, queue_manager) visitor = QueueVisitor(storage, callback, get_response_strategy(storage)) return visitor, queue_manager, storage, consumer def decode(storage_item): storage_item = json.loads(storage_item.decode()) if not isinstance(storage_item, list): storage_item = [storage_item] yield from map(compose(star(freeze), unpack), storage_item) @pytest.mark.parametrize( "one_to_many", [ False, True, ], ) @pytest.mark.parametrize( "analysis_task", [ False, True, ], ) @pytest.mark.parametrize("n_items", [2]) @pytest.mark.parametrize("n_pages", [1]) @pytest.mark.parametrize("buffer_size", [2]) @pytest.mark.parametrize( "storage_item_has_metadata", [ True, False, ], ) @pytest.mark.parametrize( "item_type", [ "string", "image", "pdf", ], ) @pytest.mark.parametrize( "queue_manager_name", [ "mock", # "pika", ], scope="session", ) @pytest.mark.parametrize( "client_name", [ "mock", # "s3", # "azure", ], scope="session", ) def test_serving( server_process, input_data_items, unencoded_input_data, metadata, bucket_name, components, core_operation, storage_item_has_metadata, target_data_items, targets, ): visitor, queue_manager, storage, consumer = components queue_manager.clear() storage.clear_bucket(bucket_name) if storage_item_has_metadata: data_metadata_packs = starmap(compose(lambda s: s.encode(), json.dumps, pack), zip(input_data_items, metadata)) else: data_metadata_packs = map(compose(lambda s: s.encode(), json.dumps, bytes_to_string), input_data_items) metadata = repeat({}) targets = lzip(target_data_items, metadata) targets = sorted(starmap(freeze, targets), key=itemgetter(0)) adorned_data_metadata_packs = adorn_data_with_storage_info(data_metadata_packs) for data, message in adorned_data_metadata_packs: storage.put_object(**get_object_descriptor(message), data=gzip.compress(data)) queue_manager.publish_request(message) reqs = consumer.consume(inactivity_timeout=5) for _, req in zip(adorned_data_metadata_packs, reqs): queue_manager.publish_response(req, visitor) names_of_uploaded_files = lfilter(".out", storage.get_all_object_names(bucket_name)) uploaded_files = [storage.get_object(bucket_name, fn) for fn in names_of_uploaded_files] outputs = sorted(chain(*map(decode, uploaded_files)), key=itemgetter(0)) assert outputs == targets