From 261b991049cbd1798b04321dc10c491de92bae70 Mon Sep 17 00:00:00 2001 From: Julius Unverfehrt Date: Tue, 5 Sep 2023 12:48:30 +0200 Subject: [PATCH] Add stacktrace to processing failures If a processing failure occures in the processing callback, pyinfra now additionally to the exeption prints the stack trace. Also removes knutils logging for now, since it still contains bugs and it should be tested first in a non-production environment if production-readiness is given. --- poetry.lock | 79 ++++--------------------- pyinfra/k8s_probes/startup.py | 2 +- pyinfra/payload_processing/processor.py | 2 +- pyinfra/queue/queue_manager.py | 4 +- pyinfra/storage/storage_provider.py | 2 +- pyinfra/storage/storages/azure.py | 2 +- pyinfra/storage/storages/s3.py | 2 +- pyproject.toml | 4 +- 8 files changed, 20 insertions(+), 77 deletions(-) diff --git a/poetry.lock b/poetry.lock index d591c40..f6cc968 100644 --- a/poetry.lock +++ b/poetry.lock @@ -32,17 +32,17 @@ wrapt = [ [[package]] name = "asttokens" -version = "2.2.1" +version = "2.4.0" description = "Annotate AST trees with source code positions" optional = false python-versions = "*" files = [ - {file = "asttokens-2.2.1-py2.py3-none-any.whl", hash = "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"}, - {file = "asttokens-2.2.1.tar.gz", hash = "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3"}, + {file = "asttokens-2.4.0-py2.py3-none-any.whl", hash = "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"}, + {file = "asttokens-2.4.0.tar.gz", hash = "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e"}, ] [package.dependencies] -six = "*" +six = ">=1.12.0" [package.extras] test = ["astroid", "pytest"] @@ -554,27 +554,6 @@ websocket-client = ">=0.32.0" [package.extras] ssh = ["paramiko (>=2.4.3)"] -[[package]] -name = "dynaconf" -version = "3.2.2" -description = "The dynamic configurator for your Python Project" -optional = false -python-versions = ">=3.8" -files = [ - {file = "dynaconf-3.2.2-py2.py3-none-any.whl", hash = "sha256:0d62e51af6e9971e8e45cabee487ec70467d6c5065a9f070beac973bedaf1d54"}, - {file = "dynaconf-3.2.2.tar.gz", hash = "sha256:2f98ec85a2b8edb767b3ed0f82c6d605d30af116ce4622932a719ba70ff152fc"}, -] - -[package.extras] -all = ["configobj", "hvac", "redis", "ruamel.yaml"] -configobj = ["configobj"] -ini = ["configobj"] -redis = ["redis"] -test = ["configobj", "django", "flake8", "flake8-debugger", "flake8-print", "flake8-todo", "flask (>=0.12)", "hvac", "pep8-naming", "pytest", "pytest-cov", "pytest-mock", "pytest-xdist", "python-dotenv", "radon", "redis", "toml"] -toml = ["toml"] -vault = ["hvac"] -yaml = ["ruamel.yaml"] - [[package]] name = "exceptiongroup" version = "1.1.3" @@ -657,13 +636,13 @@ files = [ [[package]] name = "ipykernel" -version = "6.25.1" +version = "6.25.2" description = "IPython Kernel for Jupyter" optional = false python-versions = ">=3.8" files = [ - {file = "ipykernel-6.25.1-py3-none-any.whl", hash = "sha256:c8a2430b357073b37c76c21c52184db42f6b4b0e438e1eb7df3c4440d120497c"}, - {file = "ipykernel-6.25.1.tar.gz", hash = "sha256:050391364c0977e768e354bdb60cbbfbee7cbb943b1af1618382021136ffd42f"}, + {file = "ipykernel-6.25.2-py3-none-any.whl", hash = "sha256:2e2ee359baba19f10251b99415bb39de1e97d04e1fab385646f24f0596510b77"}, + {file = "ipykernel-6.25.2.tar.gz", hash = "sha256:f468ddd1f17acb48c8ce67fcfa49ba6d46d4f9ac0438c1f441be7c3d1372230b"}, ] [package.dependencies] @@ -820,28 +799,6 @@ traitlets = ">=5.3" docs = ["myst-parser", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling", "traitlets"] test = ["ipykernel", "pre-commit", "pytest", "pytest-cov", "pytest-timeout"] -[[package]] -name = "kn-utils" -version = "0.1.5" -description = "Shared code related to logging for research & development." -optional = false -python-versions = ">=3.8,<4.0" -files = [ - {file = "kn_utils-0.1.5-py3-none-any.whl", hash = "sha256:f0d0c97ae3b4f960f1c9a008de11b1c040dfd6f38330d84414d0d0a2f146872b"}, - {file = "kn_utils-0.1.5.tar.gz", hash = "sha256:8655394afcd7583fc8cfa385e643252cb2aac5b4262603ff6fe43a60d74c958a"}, -] - -[package.dependencies] -dynaconf = ">=3.1.12,<4.0.0" -funcy = ">=1.17,<2.0" -loguru = ">=0.6,<0.7" -pytest-loguru = ">=0.2.0,<0.3.0" - -[package.source] -type = "legacy" -url = "https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi/simple" -reference = "gitlab-research" - [[package]] name = "lazy-object-proxy" version = "1.9.0" @@ -1262,13 +1219,13 @@ testutils = ["gitpython (>3)"] [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.1-py3-none-any.whl", hash = "sha256:460c9a59b14e27c602eb5ece2e47bec99dc5fc5f6513cf924a7d03a578991b1f"}, + {file = "pytest-7.4.1.tar.gz", hash = "sha256:2f2301e797521b23e4d2585a0a3d7b5e50fdddaaf7e7d6773ea26ddb17c213ab"}, ] [package.dependencies] @@ -1282,20 +1239,6 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] -[[package]] -name = "pytest-loguru" -version = "0.2.0" -description = "Pytest Loguru" -optional = false -python-versions = ">=3.6" -files = [ - {file = "pytest-loguru-0.2.0.tar.gz", hash = "sha256:6588efbc5d4ab87b05a9a37fdd0be6464d290dc985bc2fb0e5b8425fca7fb93a"}, -] - -[package.dependencies] -loguru = "*" -pytest = "*" - [[package]] name = "python-dateutil" version = "2.8.2" @@ -1765,4 +1708,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "b515275e28b523320cd324f1c29fca5d01866ca0b2ebcf47d147b89a3f5a1429" +content-hash = "ea02a1ca00a8d3063dd400399cc694e934c135f99ee8c55486d7557e1d49ec9f" diff --git a/pyinfra/k8s_probes/startup.py b/pyinfra/k8s_probes/startup.py index 9a8a183..bd1c027 100644 --- a/pyinfra/k8s_probes/startup.py +++ b/pyinfra/k8s_probes/startup.py @@ -1,5 +1,5 @@ import sys -from kn_utils.logging import logger +from loguru import logger from pathlib import Path from pyinfra.queue.queue_manager import token_file_name diff --git a/pyinfra/payload_processing/processor.py b/pyinfra/payload_processing/processor.py index 4fcea1d..90f915d 100644 --- a/pyinfra/payload_processing/processor.py +++ b/pyinfra/payload_processing/processor.py @@ -1,4 +1,4 @@ -from kn_utils.logging import logger +from loguru import logger from dataclasses import asdict from typing import Callable, List diff --git a/pyinfra/queue/queue_manager.py b/pyinfra/queue/queue_manager.py index 9a8e8b8..aa85c1c 100644 --- a/pyinfra/queue/queue_manager.py +++ b/pyinfra/queue/queue_manager.py @@ -5,7 +5,7 @@ import logging import pika import pika.exceptions import signal -from kn_utils.logging import logger +from loguru import logger from pathlib import Path from pika.adapters.blocking_connection import BlockingChannel @@ -193,7 +193,7 @@ class QueueManager: except ProcessingFailure as err: logger.info(f"Processing message with delivery_tag {frame.delivery_tag} failed, declining.") - logger.debug(f"ProcessingFailure: {err}") + logger.exception(err) self._channel.basic_nack(frame.delivery_tag, requeue=False) except Exception: diff --git a/pyinfra/storage/storage_provider.py b/pyinfra/storage/storage_provider.py index 345a096..c2c45c0 100644 --- a/pyinfra/storage/storage_provider.py +++ b/pyinfra/storage/storage_provider.py @@ -1,6 +1,6 @@ from dataclasses import asdict from functools import partial, lru_cache -from kn_utils.logging import logger +from loguru import logger from typing import Tuple from pyinfra.config import Config diff --git a/pyinfra/storage/storages/azure.py b/pyinfra/storage/storages/azure.py index 4602da9..fcbd0b5 100644 --- a/pyinfra/storage/storages/azure.py +++ b/pyinfra/storage/storages/azure.py @@ -1,7 +1,7 @@ import logging from azure.storage.blob import BlobServiceClient, ContainerClient from itertools import repeat -from kn_utils.logging import logger +from loguru import logger from operator import attrgetter from retry import retry diff --git a/pyinfra/storage/storages/s3.py b/pyinfra/storage/storages/s3.py index cdf35af..1f47618 100644 --- a/pyinfra/storage/storages/s3.py +++ b/pyinfra/storage/storages/s3.py @@ -1,6 +1,6 @@ import io from itertools import repeat -from kn_utils.logging import logger +from loguru import logger from minio import Minio from operator import attrgetter from retry import retry diff --git a/pyproject.toml b/pyproject.toml index a2c843d..76cc93d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyinfra" -version = "1.7.0" +version = "1.8.0" description = "" authors = ["Team Research "] license = "All rights reseverd" @@ -16,7 +16,7 @@ azure-storage-blob = "^12.9.0" funcy = "^1.17" prometheus-client = "^0.16.0" pycryptodome = "^3.17" -kn-utils = {version = "^0.1.5", source = "gitlab-research"} +loguru = "^0.6.0" [tool.poetry.group.dev.dependencies] pytest = "^7.1.3"