From c2d7127a842b2baf90dc87ed566c79e1a164b70a Mon Sep 17 00:00:00 2001 From: Julius Unverfehrt Date: Thu, 2 Jun 2022 14:14:54 +0200 Subject: [PATCH] add log for Consumer Error, fix page index hash function --- pyinfra/visitor.py | 2 +- scripts/mock_client.py | 2 +- src/serve.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyinfra/visitor.py b/pyinfra/visitor.py index c4e6b2b..ddc8bf2 100644 --- a/pyinfra/visitor.py +++ b/pyinfra/visitor.py @@ -19,7 +19,7 @@ from pyinfra.storage.storage import Storage def unique_hash(pages, seed=""): assert isinstance(seed, str) - pages_str = "-".join(pages) + pages_str = "-".join(map(str, pages)) seed = seed or str(time.time()) rand_str = (pages_str + seed).encode(encoding="UTF-8", errors="strict") hsh = hashlib.md5(rand_str).hexdigest() diff --git a/scripts/mock_client.py b/scripts/mock_client.py index 8a2624c..96eacd9 100644 --- a/scripts/mock_client.py +++ b/scripts/mock_client.py @@ -48,7 +48,7 @@ def build_message_bodies(analyse_container_type, bucket_name): if analyse_container_type == "detr" or analyse_container_type == "image": message_dict.update({"targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "IMAGE_INFO.json.gz"}) if analyse_container_type == "conversion": - message_dict.update({"targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "png.gz", "operation": "conversion"}) + message_dict.update({"targetFileExtension": "ORIGIN.pdf.gz", "responseFileExtension": "png.gz", "operation": "conversion", "pages": [1,2,3]}) if analyse_container_type == "ner": message_dict.update( {"targetFileExtension": "TEXT.json.gz", "responseFileExtension": "NER_ENTITIES.json.gz"} diff --git a/src/serve.py b/src/serve.py index 4c69f7b..f38b9a8 100644 --- a/src/serve.py +++ b/src/serve.py @@ -10,12 +10,16 @@ from pyinfra.flask import run_probing_webserver, set_up_probing_webserver from pyinfra.utils.banner import show_banner +logger = logging.getLogger() + + @retry(ConsumerError, tries=3, delay=5, jitter=(1, 3)) def consume(): consumer = get_consumer() try: consumer.basic_consume_and_publish() except Exception as err: + logger.exception(err) raise ConsumerError() from err @@ -44,4 +48,4 @@ if __name__ == "__main__": logging.getLogger("flask").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) - main() \ No newline at end of file + main()