diff --git a/README.md b/README.md index 5f4b5e3..98ac178 100755 --- a/README.md +++ b/README.md @@ -61,6 +61,13 @@ the [complete example](pyinfra/examples.py). | TRACING__ENDPOINT | tracing.endpoint | Endpoint to which OpenTelemetry traces are exported | TRACING__SERVER_NAME | tracing.server_name | Name of the service as displayed in the traces collected +### OpenTelemetry + +Open telemetry (vis its Python SDK) is set up to be as unobtrusive as possible; for typical use cases it can be configured +from environment variables, without additional work in the microservice app, although additional confiuration is possible. + +`TRACING_ENDPOINT` should typically be set to `http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces`. + ## Queue Manager The queue manager is responsible for consuming messages from the input queue, processing them and sending the response diff --git a/pyinfra/examples.py b/pyinfra/examples.py index 21a82a7..4e62e69 100644 --- a/pyinfra/examples.py +++ b/pyinfra/examples.py @@ -2,15 +2,18 @@ from dynaconf import Dynaconf from fastapi import FastAPI from kn_utils.logging import logger -from pyinfra.config.loader import validate_settings, get_all_validators -from pyinfra.queue.callback import make_download_process_upload_callback, DataProcessor +from pyinfra.config.loader import get_all_validators, validate_settings +from pyinfra.queue.callback import DataProcessor, make_download_process_upload_callback from pyinfra.queue.manager import QueueManager -from pyinfra.utils.opentelemetry import setup_trace, instrument_pika +from pyinfra.utils.opentelemetry import instrument_pika, setup_trace from pyinfra.webserver.prometheus import ( add_prometheus_endpoint, make_prometheus_processing_time_decorator_from_settings, ) -from pyinfra.webserver.utils import add_health_check_endpoint, create_webserver_thread_from_settings +from pyinfra.webserver.utils import ( + add_health_check_endpoint, + create_webserver_thread_from_settings, +) def start_standard_queue_consumer( diff --git a/pyinfra/queue/callback.py b/pyinfra/queue/callback.py index 2b39a45..997c3f4 100644 --- a/pyinfra/queue/callback.py +++ b/pyinfra/queue/callback.py @@ -4,7 +4,10 @@ from dynaconf import Dynaconf from kn_utils.logging import logger from pyinfra.storage.connection import get_storage -from pyinfra.storage.utils import download_data_as_specified_in_message, upload_data_as_specified_in_message +from pyinfra.storage.utils import ( + download_data_as_specified_in_message, + upload_data_as_specified_in_message, +) DataProcessor = Callable[[Union[dict, bytes], dict], dict] diff --git a/pyinfra/storage/connection.py b/pyinfra/storage/connection.py index 75d278a..442463c 100644 --- a/pyinfra/storage/connection.py +++ b/pyinfra/storage/connection.py @@ -4,12 +4,15 @@ import requests from dynaconf import Dynaconf from kn_utils.logging import logger +from pyinfra.config.loader import validate_settings +from pyinfra.config.validators import ( + multi_tenant_storage_validators, + storage_validators, +) from pyinfra.storage.storages.azure import get_azure_storage_from_settings from pyinfra.storage.storages.s3 import get_s3_storage_from_settings from pyinfra.storage.storages.storage import Storage from pyinfra.utils.cipher import decrypt -from pyinfra.config.validators import storage_validators, multi_tenant_storage_validators -from pyinfra.config.loader import validate_settings def get_storage(settings: Dynaconf, tenant_id: str = None) -> Storage: diff --git a/pyinfra/storage/storages/azure.py b/pyinfra/storage/storages/azure.py index 1207d9a..4ffd2d4 100644 --- a/pyinfra/storage/storages/azure.py +++ b/pyinfra/storage/storages/azure.py @@ -7,9 +7,9 @@ from dynaconf import Dynaconf from kn_utils.logging import logger from retry import retry -from pyinfra.storage.storages.storage import Storage -from pyinfra.config.validators import azure_storage_validators from pyinfra.config.loader import validate_settings +from pyinfra.config.validators import azure_storage_validators +from pyinfra.storage.storages.storage import Storage logging.getLogger("azure").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) diff --git a/pyinfra/storage/storages/s3.py b/pyinfra/storage/storages/s3.py index 0a6636e..d84601f 100644 --- a/pyinfra/storage/storages/s3.py +++ b/pyinfra/storage/storages/s3.py @@ -7,9 +7,9 @@ from kn_utils.logging import logger from minio import Minio from retry import retry -from pyinfra.storage.storages.storage import Storage -from pyinfra.config.validators import s3_storage_validators from pyinfra.config.loader import validate_settings +from pyinfra.config.validators import s3_storage_validators +from pyinfra.storage.storages.storage import Storage from pyinfra.utils.url_parsing import validate_and_parse_s3_endpoint diff --git a/pyinfra/utils/opentelemetry.py b/pyinfra/utils/opentelemetry.py index c994ea1..b3909af 100644 --- a/pyinfra/utils/opentelemetry.py +++ b/pyinfra/utils/opentelemetry.py @@ -9,8 +9,12 @@ from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.pika import PikaInstrumentor from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.sdk.trace.export import ( + BatchSpanProcessor, + ConsoleSpanExporter, + SpanExporter, + SpanExportResult, +) from pyinfra.config.loader import validate_settings from pyinfra.config.validators import opentelemetry_validators diff --git a/pyinfra/webserver/prometheus.py b/pyinfra/webserver/prometheus.py index 4bd52dc..6a9f045 100644 --- a/pyinfra/webserver/prometheus.py +++ b/pyinfra/webserver/prometheus.py @@ -4,11 +4,11 @@ from typing import Callable, TypeVar from dynaconf import Dynaconf from fastapi import FastAPI from funcy import identity -from prometheus_client import generate_latest, CollectorRegistry, REGISTRY, Summary +from prometheus_client import REGISTRY, CollectorRegistry, Summary, generate_latest from starlette.responses import Response -from pyinfra.config.validators import prometheus_validators from pyinfra.config.loader import validate_settings +from pyinfra.config.validators import prometheus_validators def add_prometheus_endpoint(app: FastAPI, registry: CollectorRegistry = REGISTRY) -> FastAPI: @@ -37,7 +37,7 @@ def make_prometheus_processing_time_decorator_from_settings( registry: CollectorRegistry = REGISTRY, ) -> Decorator: """Make a decorator for monitoring the processing time of a function. This, and other metrics should follow the - convention {product name}_{service name}_{processing step / parameter to monitor}. + convention {product name}_{service name}_{processing step / parameter to monitor}. """ validate_settings(settings, validators=prometheus_validators) diff --git a/pyinfra/webserver/utils.py b/pyinfra/webserver/utils.py index ffe72c5..bbf5ac5 100644 --- a/pyinfra/webserver/utils.py +++ b/pyinfra/webserver/utils.py @@ -6,9 +6,9 @@ import uvicorn from dynaconf import Dynaconf from fastapi import FastAPI -from pyinfra.config.validators import webserver_validators from pyinfra.config.loader import validate_settings -from pyinfra.utils.opentelemetry import setup_trace, instrument_app +from pyinfra.config.validators import webserver_validators +from pyinfra.utils.opentelemetry import instrument_app, setup_trace def create_webserver_thread_from_settings(app: FastAPI, settings: Dynaconf) -> threading.Thread: @@ -30,7 +30,6 @@ def create_webserver_thread(app: FastAPI, port: int, host: str) -> threading.Thr def create_webserver_thread_with_tracing(app: FastAPI, settings: Dynaconf) -> threading.Thread: - def inner(): setup_trace(settings) instrument_app(app) diff --git a/tests/unit_test/knutils_logger_test.py b/tests/unit_test/knutils_logger_test.py index f082428..1248ce2 100644 --- a/tests/unit_test/knutils_logger_test.py +++ b/tests/unit_test/knutils_logger_test.py @@ -4,7 +4,7 @@ from kn_utils.logging import logger def test_necessary_log_levels_are_supported_by_kn_utils(): logger.setLevel("TRACE") - + logger.trace("trace") logger.debug("debug") logger.info("info") @@ -13,6 +13,7 @@ def test_necessary_log_levels_are_supported_by_kn_utils(): logger.exception("exception", exc_info="this is an exception") logger.error("error", exc_info="this is an error") + def test_setlevel_warn(): logger.setLevel("WARN") logger.warning("warn") diff --git a/tests/unit_test/opentelemetry_test.py b/tests/unit_test/opentelemetry_test.py index e624b18..3a68ab9 100644 --- a/tests/unit_test/opentelemetry_test.py +++ b/tests/unit_test/opentelemetry_test.py @@ -2,7 +2,7 @@ from time import sleep import pytest -from pyinfra.utils.opentelemetry import get_exporter, setup_trace, instrument_pika +from pyinfra.utils.opentelemetry import get_exporter, instrument_pika, setup_trace @pytest.fixture(scope="session") diff --git a/tests/unit_test/prometheus_monitoring_test.py b/tests/unit_test/prometheus_monitoring_test.py index 3ced056..37bf458 100644 --- a/tests/unit_test/prometheus_monitoring_test.py +++ b/tests/unit_test/prometheus_monitoring_test.py @@ -5,7 +5,10 @@ import pytest import requests from fastapi import FastAPI -from pyinfra.webserver.prometheus import add_prometheus_endpoint, make_prometheus_processing_time_decorator_from_settings +from pyinfra.webserver.prometheus import ( + add_prometheus_endpoint, + make_prometheus_processing_time_decorator_from_settings, +) from pyinfra.webserver.utils import create_webserver_thread_from_settings diff --git a/tests/unit_test/storage_test.py b/tests/unit_test/storage_test.py index 365d066..6f69be9 100644 --- a/tests/unit_test/storage_test.py +++ b/tests/unit_test/storage_test.py @@ -6,7 +6,10 @@ import pytest from fastapi import FastAPI from pyinfra.storage.connection import get_storage_from_tenant_id -from pyinfra.storage.utils import download_data_as_specified_in_message, upload_data_as_specified_in_message +from pyinfra.storage.utils import ( + download_data_as_specified_in_message, + upload_data_as_specified_in_message, +) from pyinfra.utils.cipher import encrypt from pyinfra.webserver.utils import create_webserver_thread