From dccb9ac67a7559ee382131bb5c229fb93af85ee0 Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Wed, 23 Feb 2022 11:41:14 +0100 Subject: [PATCH] Pull request #10: Endpoint config changes Merge in RR/pyinfra from endpoint_config_changes to master Squashed commit of the following: commit 4392513d1b542ff8b3ca4702827ad656cb27ab4c Author: Matthias Bisping Date: Tue Feb 22 13:50:53 2022 +0100 fixed comment in config commit 1ee27db1536ec085fdc3d249aec16bd69a83d73a Author: Matthias Bisping Date: Tue Feb 22 13:47:58 2022 +0100 fixed minio endpoint in config commit bc07d5086f30dafc9adf4e6eb6002f92ee3c11a5 Author: Matthias Bisping Date: Tue Feb 22 13:31:54 2022 +0100 config refac commit 2ad7fc6477cd986647b44fde8a3c0ec5938ec203 Author: Matthias Bisping Date: Tue Feb 22 13:21:56 2022 +0100 refactorig of webserver startup logic --- config.yaml | 16 ++++++++++------ pyinfra/flask.py | 41 ++++++++++++++++++++++++++++++++++++++++ pyinfra/storage/minio.py | 2 +- src/serve.py | 26 ++----------------------- 4 files changed, 54 insertions(+), 31 deletions(-) create mode 100644 pyinfra/flask.py diff --git a/config.yaml b/config.yaml index 1c0b44b..5cbe777 100755 --- a/config.yaml +++ b/config.yaml @@ -17,18 +17,22 @@ rabbitmq: max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue minio: - host: $STORAGE_ENDPOINT|localhost # MinIO host address - port: $STORAGE_PORT|9000 # MinIO host port + endpoint: $STORAGE_ENDPOINT|"127.0.0.1:9000" # MinIO endpoint user: $STORAGE_KEY|root # MinIO user name password: $STORAGE_SECRET|password # MinIO user password bucket: $STORAGE_BUCKET_NAME|redaction # MinIO bucket +azure_blob_storage: + connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net" + container: $STORAGE_AZURECONTAINERNAME|"pyinfra-test-storage" + service: - logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for log file messages + logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger name: $SERVICE_NAME|pyinfra-service-v1 # Name of the service in the kubernetes cluster storage_backend: $STORAGE_BACKEND|s3 # The storage to pull files to be processed from analysis_endpoint: $ANALYSIS_ENDPOINT|"http://127.0.0.1:5000" -webserver: - host: $PROBE_SERVER_HOST|"0.0.0.0" # Probe webserver address - port: $PROBE_SERVER_PORT|8080 # Probe webserver port +probing_webserver: + host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address + port: $PROBING_WEBSERVER_PORT|8080 # Probe webserver port + mode: $PROBING_WEBSERVER_MODE|production # webserver mode: {development, production} diff --git a/pyinfra/flask.py b/pyinfra/flask.py new file mode 100644 index 0000000..e8eac08 --- /dev/null +++ b/pyinfra/flask.py @@ -0,0 +1,41 @@ +from flask import Flask, jsonify +from waitress import serve + +from pyinfra.config import CONFIG + + +def run_probing_webserver(app, host=None, port=None, mode=None): + + if not host: + host = CONFIG.probing_webserver.host + + if not port: + port = CONFIG.probing_webserver.port + + if not mode: + mode = CONFIG.probing_webserver.mode + + if mode == "development": + app.run(host=host, port=port, debug=True) + + elif mode == "production": + serve(app, host=host, port=port) + + +def set_up_probing_webserver(): + + app = Flask(__name__) + + @app.route("/ready", methods=["GET"]) + def ready(): + resp = jsonify("OK") + resp.status_code = 200 + return resp + + @app.route("/health", methods=["GET"]) + def healthy(): + resp = jsonify("OK") + resp.status_code = 200 + return resp + + return app diff --git a/pyinfra/storage/minio.py b/pyinfra/storage/minio.py index 681283d..5f4f4b4 100644 --- a/pyinfra/storage/minio.py +++ b/pyinfra/storage/minio.py @@ -22,7 +22,7 @@ def get_minio_client(access_key=None, secret_key=None) -> Minio: secret_key = CONFIG.minio.password if secret_key is None else secret_key # TODO: secure=True/False? - return Minio(f"{CONFIG.minio.host}:{CONFIG.minio.port}", access_key=access_key, secret_key=secret_key, secure=False) + return Minio(CONFIG.minio.endpoint, access_key=access_key, secret_key=secret_key, secure=False) class MinioHandle(StorageHandle): diff --git a/src/serve.py b/src/serve.py index 5c3d3c1..a37f3b4 100644 --- a/src/serve.py +++ b/src/serve.py @@ -2,8 +2,6 @@ import logging from multiprocessing import Process import pika -from flask import Flask, jsonify -from waitress import serve from pyinfra.callback import ( make_retry_callback_for_output_queue, @@ -14,31 +12,12 @@ from pyinfra.config import CONFIG from pyinfra.consume import consume, ConsumerError from pyinfra.core import make_payload_processor, make_storage_data_loader, make_analyzer from pyinfra.exceptions import UnknownStorageBackend +from pyinfra.flask import run_probing_webserver, set_up_probing_webserver from pyinfra.storage.azure_blob_storage import AzureBlobStorageHandle from pyinfra.storage.minio import MinioHandle # TODO: implement meaningful checks -def start_integrity_checks_webserver(mode="debug"): - - app = Flask(__name__) - - @app.route("/ready", methods=["GET"]) - def ready(): - resp = jsonify("OK") - resp.status_code = 200 - return resp - - @app.route("/health", methods=["GET"]) - def healthy(): - resp = jsonify("OK") - resp.status_code = 200 - return resp - - if mode == "debug": - app.run(host=CONFIG.webserver.host, port=CONFIG.webserver.port, debug=True) - elif mode == "production": - serve(app, host=CONFIG.webserver.host, port=CONFIG.webserver.port) def get_storage(): @@ -87,8 +66,7 @@ def make_callback(): def main(): - - webserver = Process(target=start_integrity_checks_webserver, args=("production",)) + webserver = Process(target=run_probing_webserver, args=(set_up_probing_webserver(),)) logging.info("Starting webserver...") webserver.start()