import gzip import json from operator import itemgetter from kn_utils.logging import logger from pyinfra.config.loader import load_settings, local_pyinfra_root_path from pyinfra.queue.manager import QueueManager from pyinfra.storage.storages.s3 import get_s3_storage_from_settings settings = load_settings(local_pyinfra_root_path / "config/") def upload_json_and_make_message_body(): dossier_id, file_id, suffix = "dossier", "file", "json.gz" content = { "numberOfPages": 7, "sectionTexts": "data", } object_name = f"{dossier_id}/{file_id}.{suffix}" data = gzip.compress(json.dumps(content).encode("utf-8")) storage = get_s3_storage_from_settings(settings) if not storage.has_bucket(): storage.make_bucket() storage.put_object(object_name, data) message_body = { "dossierId": dossier_id, "fileId": file_id, "targetFileExtension": suffix, "responseFileExtension": f"result.{suffix}", } return message_body def main(): queue_manager = QueueManager(settings) queue_manager.purge_queues() message = upload_json_and_make_message_body() queue_manager.publish_message_to_input_queue(message) logger.info(f"Put {message} on {settings.rabbitmq.input_queue}.") storage = get_s3_storage_from_settings(settings) for method_frame, properties, body in queue_manager.channel.consume( queue=settings.rabbitmq.output_queue, inactivity_timeout=15 ): if not body: break response = json.loads(body) logger.info(f"Received {response}") logger.info(f"Message headers: {properties.headers}") queue_manager.channel.basic_ack(method_frame.delivery_tag) dossier_id, file_id = itemgetter("dossierId", "fileId")(response) suffix = message["responseFileExtension"] print(f"{dossier_id}/{file_id}.{suffix}") result = storage.get_object(f"{dossier_id}/{file_id}.{suffix}") result = json.loads(gzip.decompress(result)) logger.info(f"Contents of result on storage: {result}") queue_manager.stop_consuming() if __name__ == "__main__": main()