Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

Pull request #10: Endpoint config changes

Browse Source 
Merge in RR/pyinfra from endpoint_config_changes to master

Squashed commit of the following:

commit 4392513d1b542ff8b3ca4702827ad656cb27ab4c
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date:   Tue Feb 22 13:50:53 2022 +0100

    fixed comment in config

commit 1ee27db1536ec085fdc3d249aec16bd69a83d73a
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date:   Tue Feb 22 13:47:58 2022 +0100

    fixed minio endpoint in config

commit bc07d5086f30dafc9adf4e6eb6002f92ee3c11a5
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date:   Tue Feb 22 13:31:54 2022 +0100

    config refac

commit 2ad7fc6477cd986647b44fde8a3c0ec5938ec203
Author: Matthias Bisping <matthias.bisping@iqser.com>
Date:   Tue Feb 22 13:21:56 2022 +0100

    refactorig of webserver startup logic
This commit is contained in:
Matthias Bisping 2022-02-23 11:41:14 +01:00
parent daaa0ed958
commit dccb9ac67a
4 changed files with 54 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
config.yaml
View File

@ -17,18 +17,22 @@ rabbitmq:
max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue
minio:
host: $STORAGE_ENDPOINT|localhost # MinIO host address
port: $STORAGE_PORT|9000 # MinIO host port
endpoint: $STORAGE_ENDPOINT|"127.0.0.1:9000" # MinIO endpoint
user: $STORAGE_KEY|root # MinIO user name
password: $STORAGE_SECRET|password # MinIO user password
bucket: $STORAGE_BUCKET_NAME|redaction # MinIO bucket
azure_blob_storage:
connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
container: $STORAGE_AZURECONTAINERNAME|"pyinfra-test-storage"
service:
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for log file messages
logging_level: $LOGGING_LEVEL_ROOT|DEBUG # Logging level for service logger
name: $SERVICE_NAME|pyinfra-service-v1 # Name of the service in the kubernetes cluster
storage_backend: $STORAGE_BACKEND|s3 # The storage to pull files to be processed from
analysis_endpoint: $ANALYSIS_ENDPOINT|"http://127.0.0.1:5000"
webserver:
host: $PROBE_SERVER_HOST|"0.0.0.0" # Probe webserver address
port: $PROBE_SERVER_PORT|8080 # Probe webserver port
probing_webserver:
host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address
port: $PROBING_WEBSERVER_PORT|8080 # Probe webserver port
mode: $PROBING_WEBSERVER_MODE|production # webserver mode: {development, production}

41
pyinfra/flask.py Normal file
View File

@ -0,0 +1,41 @@
from flask import Flask, jsonify
from waitress import serve
from pyinfra.config import CONFIG
def run_probing_webserver(app, host=None, port=None, mode=None):
if not host:
host = CONFIG.probing_webserver.host
if not port:
port = CONFIG.probing_webserver.port
if not mode:
mode = CONFIG.probing_webserver.mode
if mode == "development":
app.run(host=host, port=port, debug=True)
elif mode == "production":
serve(app, host=host, port=port)
def set_up_probing_webserver():
app = Flask(__name__)
@app.route("/ready", methods=["GET"])
def ready():
resp = jsonify("OK")
resp.status_code = 200
return resp
@app.route("/health", methods=["GET"])
def healthy():
resp = jsonify("OK")
resp.status_code = 200
return resp
return app

2
pyinfra/storage/minio.py
View File

@ -22,7 +22,7 @@ def get_minio_client(access_key=None, secret_key=None) -> Minio:
secret_key = CONFIG.minio.password if secret_key is None else secret_key
# TODO: secure=True/False?
return Minio(f"{CONFIG.minio.host}:{CONFIG.minio.port}", access_key=access_key, secret_key=secret_key, secure=False)
return Minio(CONFIG.minio.endpoint, access_key=access_key, secret_key=secret_key, secure=False)
class MinioHandle(StorageHandle):

26
src/serve.py
View File

@ -2,8 +2,6 @@ import logging
from multiprocessing import Process
import pika
from flask import Flask, jsonify
from waitress import serve
from pyinfra.callback import (
make_retry_callback_for_output_queue,
@ -14,31 +12,12 @@ from pyinfra.config import CONFIG
from pyinfra.consume import consume, ConsumerError
from pyinfra.core import make_payload_processor, make_storage_data_loader, make_analyzer
from pyinfra.exceptions import UnknownStorageBackend
from pyinfra.flask import run_probing_webserver, set_up_probing_webserver
from pyinfra.storage.azure_blob_storage import AzureBlobStorageHandle
from pyinfra.storage.minio import MinioHandle
# TODO: implement meaningful checks
def start_integrity_checks_webserver(mode="debug"):
app = Flask(__name__)
@app.route("/ready", methods=["GET"])
def ready():
resp = jsonify("OK")
resp.status_code = 200
return resp
@app.route("/health", methods=["GET"])
def healthy():
resp = jsonify("OK")
resp.status_code = 200
return resp
if mode == "debug":
app.run(host=CONFIG.webserver.host, port=CONFIG.webserver.port, debug=True)
elif mode == "production":
serve(app, host=CONFIG.webserver.host, port=CONFIG.webserver.port)
def get_storage():
@ -87,8 +66,7 @@ def make_callback():
def main():
webserver = Process(target=start_integrity_checks_webserver, args=("production",))
webserver = Process(target=run_probing_webserver, args=(set_up_probing_webserver(),))
logging.info("Starting webserver...")
webserver.start()