08f2746afc
Merge in RR/pyinfra from prometheus-tunneling to pika_encapsulation
Squashed commit of the following:
commit 350448ec0f14849844deaa1a86ba4397ab3ebf3c
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Wed Mar 9 16:52:02 2022 +0100
quickfix
commit ee88be4c80abdf597013c743ce2d490ad2b3e029
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Wed Mar 9 15:26:19 2022 +0100
added prometheus endpoint to tunnel metrics from analysis endpoint
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
import logging
|
|
from multiprocessing import Process
|
|
|
|
import pika
|
|
|
|
from pyinfra.callback import (
|
|
make_retry_callback_for_output_queue,
|
|
make_retry_callback,
|
|
make_callback_for_output_queue,
|
|
)
|
|
from pyinfra.config import CONFIG, make_art
|
|
from pyinfra.consume import consume, ConsumerError
|
|
from pyinfra.core import make_payload_processor, make_storage_data_loader, make_analyzer
|
|
from pyinfra.flask import run_probing_webserver, set_up_probing_webserver
|
|
from pyinfra.storage.storages import get_storage
|
|
|
|
|
|
def republish(channel, body, n_current_attempts):
|
|
channel.basic_publish(
|
|
exchange="",
|
|
routing_key=CONFIG.rabbitmq.queues.input,
|
|
body=body,
|
|
properties=pika.BasicProperties(headers={"x-retry-count": n_current_attempts}),
|
|
)
|
|
|
|
|
|
def make_callback():
|
|
|
|
load_data = make_storage_data_loader(get_storage(CONFIG.storage.backend), CONFIG.storage.bucket)
|
|
analyze_file = make_analyzer(CONFIG.rabbitmq.callback.analysis_endpoint)
|
|
|
|
json_wrapped_body_processor = make_payload_processor(load_data, analyze_file)
|
|
|
|
if CONFIG.rabbitmq.callback.retry.enabled:
|
|
retry_callback = make_retry_callback(republish, max_attempts=CONFIG.rabbitmq.callback.retry.max_attempts)
|
|
|
|
callback = make_retry_callback_for_output_queue(
|
|
json_wrapped_body_processor=json_wrapped_body_processor,
|
|
output_queue_name=CONFIG.rabbitmq.queues.output,
|
|
retry_callback=retry_callback,
|
|
)
|
|
else:
|
|
callback = make_callback_for_output_queue(
|
|
json_wrapped_body_processor=json_wrapped_body_processor, output_queue_name=CONFIG.rabbitmq.queues.output
|
|
)
|
|
|
|
return callback
|
|
|
|
|
|
def main():
|
|
# TODO: implement meaningful checks
|
|
logging.info(make_art())
|
|
webserver = Process(target=run_probing_webserver, args=(set_up_probing_webserver(),))
|
|
logging.info("Starting webserver...")
|
|
webserver.start()
|
|
|
|
try:
|
|
consume(CONFIG.rabbitmq.queues.input, make_callback())
|
|
except KeyboardInterrupt:
|
|
pass
|
|
except ConsumerError:
|
|
webserver.terminate()
|
|
raise
|
|
|
|
webserver.join()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging_level = CONFIG.service.logging_level
|
|
logging.basicConfig(level=logging_level)
|
|
logging.getLogger("pika").setLevel(logging.ERROR)
|
|
logging.getLogger("flask").setLevel(logging.ERROR)
|
|
logging.getLogger("urllib3").setLevel(logging.ERROR)
|
|
|
|
main()
|