chore: add opentelemetry subsection to README.md; formatting

README.md

@@ -61,6 +61,13 @@ the [complete example](pyinfra/examples.py).
 | TRACING__ENDPOINT                  | tracing.endpoint                 | Endpoint to which OpenTelemetry traces are exported
 | TRACING__SERVER_NAME               | tracing.server_name              | Name of the service as displayed in the traces collected
 
+### OpenTelemetry
+
+Open telemetry (vis its Python SDK) is set up to be as unobtrusive as possible; for typical use cases it can be configured
+from environment variables, without additional work in the microservice app, although additional confiuration is possible.
+
+`TRACING_ENDPOINT` should typically be set to `http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces`.
+
 ## Queue Manager
 
 The queue manager is responsible for consuming messages from the input queue, processing them and sending the response

pyinfra/examples.py

@@ -2,15 +2,18 @@ from dynaconf import Dynaconf
 from fastapi import FastAPI
 from kn_utils.logging import logger
 
-from pyinfra.config.loader import validate_settings, get_all_validators
-from pyinfra.queue.callback import make_download_process_upload_callback, DataProcessor
+from pyinfra.config.loader import get_all_validators, validate_settings
+from pyinfra.queue.callback import DataProcessor, make_download_process_upload_callback
 from pyinfra.queue.manager import QueueManager
-from pyinfra.utils.opentelemetry import setup_trace, instrument_pika
+from pyinfra.utils.opentelemetry import instrument_pika, setup_trace
 from pyinfra.webserver.prometheus import (
     add_prometheus_endpoint,
     make_prometheus_processing_time_decorator_from_settings,
 )
-from pyinfra.webserver.utils import add_health_check_endpoint, create_webserver_thread_from_settings
+from pyinfra.webserver.utils import (
+    add_health_check_endpoint,
+    create_webserver_thread_from_settings,
+)
 
 
 def start_standard_queue_consumer(

pyinfra/queue/callback.py

@@ -4,7 +4,10 @@ from dynaconf import Dynaconf
 from kn_utils.logging import logger
 
 from pyinfra.storage.connection import get_storage
-from pyinfra.storage.utils import download_data_as_specified_in_message, upload_data_as_specified_in_message
+from pyinfra.storage.utils import (
+    download_data_as_specified_in_message,
+    upload_data_as_specified_in_message,
+)
 
 DataProcessor = Callable[[Union[dict, bytes], dict], dict]
 

pyinfra/storage/connection.py

@@ -4,12 +4,15 @@ import requests
 from dynaconf import Dynaconf
 from kn_utils.logging import logger
 
+from pyinfra.config.loader import validate_settings
+from pyinfra.config.validators import (
+    multi_tenant_storage_validators,
+    storage_validators,
+)
 from pyinfra.storage.storages.azure import get_azure_storage_from_settings
 from pyinfra.storage.storages.s3 import get_s3_storage_from_settings
 from pyinfra.storage.storages.storage import Storage
 from pyinfra.utils.cipher import decrypt
-from pyinfra.config.validators import storage_validators, multi_tenant_storage_validators
-from pyinfra.config.loader import validate_settings
 
 
 def get_storage(settings: Dynaconf, tenant_id: str = None) -> Storage:

pyinfra/storage/storages/azure.py

@@ -7,9 +7,9 @@ from dynaconf import Dynaconf
 from kn_utils.logging import logger
 from retry import retry
 
-from pyinfra.storage.storages.storage import Storage
-from pyinfra.config.validators import azure_storage_validators
 from pyinfra.config.loader import validate_settings
+from pyinfra.config.validators import azure_storage_validators
+from pyinfra.storage.storages.storage import Storage
 
 logging.getLogger("azure").setLevel(logging.WARNING)
 logging.getLogger("urllib3").setLevel(logging.WARNING)

pyinfra/storage/storages/s3.py

@@ -7,9 +7,9 @@ from kn_utils.logging import logger
 from minio import Minio
 from retry import retry
 
-from pyinfra.storage.storages.storage import Storage
-from pyinfra.config.validators import s3_storage_validators
 from pyinfra.config.loader import validate_settings
+from pyinfra.config.validators import s3_storage_validators
+from pyinfra.storage.storages.storage import Storage
 from pyinfra.utils.url_parsing import validate_and_parse_s3_endpoint
 
 

pyinfra/utils/opentelemetry.py

@@ -9,8 +9,12 @@ from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.instrumentation.pika import PikaInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
-from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+from opentelemetry.sdk.trace.export import (
+    BatchSpanProcessor,
+    ConsoleSpanExporter,
+    SpanExporter,
+    SpanExportResult,
+)
 
 from pyinfra.config.loader import validate_settings
 from pyinfra.config.validators import opentelemetry_validators

pyinfra/webserver/prometheus.py

@@ -4,11 +4,11 @@ from typing import Callable, TypeVar
 from dynaconf import Dynaconf
 from fastapi import FastAPI
 from funcy import identity
-from prometheus_client import generate_latest, CollectorRegistry, REGISTRY, Summary
+from prometheus_client import REGISTRY, CollectorRegistry, Summary, generate_latest
 from starlette.responses import Response
 
-from pyinfra.config.validators import prometheus_validators
 from pyinfra.config.loader import validate_settings
+from pyinfra.config.validators import prometheus_validators
 
 
 def add_prometheus_endpoint(app: FastAPI, registry: CollectorRegistry = REGISTRY) -> FastAPI:
@@ -37,7 +37,7 @@ def make_prometheus_processing_time_decorator_from_settings(
     registry: CollectorRegistry = REGISTRY,
 ) -> Decorator:
     """Make a decorator for monitoring the processing time of a function. This, and other metrics should follow the
-     convention {product name}_{service name}_{processing step / parameter to monitor}.
+    convention {product name}_{service name}_{processing step / parameter to monitor}.
     """
     validate_settings(settings, validators=prometheus_validators)
 

pyinfra/webserver/utils.py

@@ -6,9 +6,9 @@ import uvicorn
 from dynaconf import Dynaconf
 from fastapi import FastAPI
 
-from pyinfra.config.validators import webserver_validators
 from pyinfra.config.loader import validate_settings
-from pyinfra.utils.opentelemetry import setup_trace, instrument_app
+from pyinfra.config.validators import webserver_validators
+from pyinfra.utils.opentelemetry import instrument_app, setup_trace
 
 
 def create_webserver_thread_from_settings(app: FastAPI, settings: Dynaconf) -> threading.Thread:
@@ -30,7 +30,6 @@ def create_webserver_thread(app: FastAPI, port: int, host: str) -> threading.Thr
 
 
 def create_webserver_thread_with_tracing(app: FastAPI, settings: Dynaconf) -> threading.Thread:
-
     def inner():
         setup_trace(settings)
         instrument_app(app)

tests/unit_test/knutils_logger_test.py

@@ -4,7 +4,7 @@ from kn_utils.logging import logger
 
 def test_necessary_log_levels_are_supported_by_kn_utils():
     logger.setLevel("TRACE")
-    
+
     logger.trace("trace")
     logger.debug("debug")
     logger.info("info")
@@ -13,6 +13,7 @@ def test_necessary_log_levels_are_supported_by_kn_utils():
     logger.exception("exception", exc_info="this is an exception")
     logger.error("error", exc_info="this is an error")
 
+
 def test_setlevel_warn():
     logger.setLevel("WARN")
     logger.warning("warn")

tests/unit_test/opentelemetry_test.py

@@ -2,7 +2,7 @@ from time import sleep
 
 import pytest
 
-from pyinfra.utils.opentelemetry import get_exporter, setup_trace, instrument_pika
+from pyinfra.utils.opentelemetry import get_exporter, instrument_pika, setup_trace
 
 
 @pytest.fixture(scope="session")

tests/unit_test/prometheus_monitoring_test.py

@@ -5,7 +5,10 @@ import pytest
 import requests
 from fastapi import FastAPI
 
-from pyinfra.webserver.prometheus import add_prometheus_endpoint, make_prometheus_processing_time_decorator_from_settings
+from pyinfra.webserver.prometheus import (
+    add_prometheus_endpoint,
+    make_prometheus_processing_time_decorator_from_settings,
+)
 from pyinfra.webserver.utils import create_webserver_thread_from_settings
 
 

tests/unit_test/storage_test.py

@@ -6,7 +6,10 @@ import pytest
 from fastapi import FastAPI
 
 from pyinfra.storage.connection import get_storage_from_tenant_id
-from pyinfra.storage.utils import download_data_as_specified_in_message, upload_data_as_specified_in_message
+from pyinfra.storage.utils import (
+    download_data_as_specified_in_message,
+    upload_data_as_specified_in_message,
+)
 from pyinfra.utils.cipher import encrypt
 from pyinfra.webserver.utils import create_webserver_thread