From 1363030b950834f0b8e2fabcc3750993db63ee03 Mon Sep 17 00:00:00 2001 From: Kresnadi Budisantoso Date: Thu, 21 Apr 2022 14:55:01 +0200 Subject: [PATCH] Pull request #27: Add parser for bucket/container env vars Merge in RR/pyinfra from kbudisantoso/configyaml-1650538128334 to master Squashed commit of the following: commit 6103b7720315aaef3d98aea8f3c817477bbf500b Merge: 69ac65a 3b91185 Author: Julius Unverfehrt Date: Thu Apr 21 14:45:10 2022 +0200 Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334 commit 69ac65ae1bd4095c797112c6f9530f0b1705277e Merge: 9a1cd07 a00ceae Author: Julius Unverfehrt Date: Thu Apr 21 14:37:34 2022 +0200 Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334 commit 9a1cd07c09e5ee2618f2c1a3c27b69c67b1eaeb0 Author: Julius Unverfehrt Date: Thu Apr 21 14:35:49 2022 +0200 test done commit e7127e8af937fe067f1f92eb688187ebbe609478 Author: Julius Unverfehrt Date: Thu Apr 21 14:32:25 2022 +0200 test commit 262957e33d19dbafb3f10b5a32c438460b966a88 Author: Julius Unverfehrt Date: Thu Apr 21 14:16:33 2022 +0200 add parser for env var storage_buckets/containers commit 3535002b4aac9f297bdbe112b04f537cef25f5c2 Author: Kresnadi Budisantoso Date: Thu Apr 21 12:48:52 2022 +0200 config.yaml online editiert mit Bitbucket --- config.yaml | 2 +- pyinfra/config.py | 16 +++++++++++++++- pyinfra/visitor.py | 7 ++++--- requirements.txt | 1 + scripts/manage_minio.py | 4 ++-- scripts/mock_client.py | 4 ++-- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/config.yaml b/config.yaml index ceb12a0..31c6571 100755 --- a/config.yaml +++ b/config.yaml @@ -23,7 +23,7 @@ rabbitmq: storage: backend: $STORAGE_BACKEND|s3 # The type of storage to use {s3, azure} - bucket: $STORAGE_BUCKET|"pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from + bucket: "STORAGE_BUCKET_NAME|STORAGE_AZURECONTAINERNAME|pyinfra-test-bucket" # The bucket / container to pull files specified in queue requests from s3: endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000" diff --git a/pyinfra/config.py b/pyinfra/config.py index 06c9cae..c306e19 100644 --- a/pyinfra/config.py +++ b/pyinfra/config.py @@ -1,7 +1,9 @@ """Implements a config object with dot-indexing syntax.""" - +import os +from itertools import chain from envyaml import EnvYAML +from funcy import first, juxt, butlast, last from pyinfra.locations import CONFIG_FILE @@ -38,3 +40,15 @@ class Config: CONFIG = Config(CONFIG_FILE) + + +def parse_disjunction_string(disjunction_string): + def try_parse_env_var(disjunction_string): + try: + return os.environ[disjunction_string] + except KeyError: + return None + + options = disjunction_string.split("|") + identifiers, fallback_value = juxt(butlast, last)(options) + return first(chain(filter(try_parse_env_var, identifiers), [fallback_value])) diff --git a/pyinfra/visitor.py b/pyinfra/visitor.py index 9e0a30f..0399789 100644 --- a/pyinfra/visitor.py +++ b/pyinfra/visitor.py @@ -5,7 +5,7 @@ import logging from operator import itemgetter from typing import Callable -from pyinfra.config import CONFIG +from pyinfra.config import CONFIG, parse_disjunction_string from pyinfra.exceptions import DataLoadingFailure from pyinfra.storage.storage import Storage @@ -23,11 +23,12 @@ def get_response_object_name(body): def get_object_descriptor(body): - return {"bucket_name": CONFIG.storage.bucket, "object_name": get_object_name(body)} + return {"bucket_name": parse_disjunction_string(CONFIG.storage.bucket), "object_name": get_object_name(body)} def get_response_object_descriptor(body): - return {"bucket_name": CONFIG.storage.bucket, "object_name": get_response_object_name(body)} + return {"bucket_name": parse_disjunction_string(CONFIG.storage.bucket), + "object_name": get_response_object_name(body)} class ResponseStrategy(abc.ABC): diff --git a/requirements.txt b/requirements.txt index 64f0c27..ba7589e 100755 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,4 @@ testcontainers==3.4.2 docker-compose==1.29.2 tqdm==4.62.3 pytest~=7.0.1 +funcy==1.17 diff --git a/scripts/manage_minio.py b/scripts/manage_minio.py index eaa921e..88e1127 100644 --- a/scripts/manage_minio.py +++ b/scripts/manage_minio.py @@ -5,7 +5,7 @@ from pathlib import Path from tqdm import tqdm -from pyinfra.config import CONFIG +from pyinfra.config import CONFIG, parse_disjunction_string from pyinfra.storage.storages import get_s3_storage @@ -51,7 +51,7 @@ def add_file_compressed(storage, bucket_name, dossier_id, path) -> None: if __name__ == "__main__": storage = get_s3_storage() - bucket_name = CONFIG.storage.bucket + bucket_name = parse_disjunction_string(CONFIG.storage.bucket) if not storage.has_bucket(bucket_name): storage.make_bucket(bucket_name) diff --git a/scripts/mock_client.py b/scripts/mock_client.py index e7699fa..0b456d9 100644 --- a/scripts/mock_client.py +++ b/scripts/mock_client.py @@ -3,7 +3,7 @@ import json import pika -from pyinfra.config import CONFIG +from pyinfra.config import CONFIG, parse_disjunction_string from pyinfra.storage.storages import get_s3_storage @@ -53,7 +53,7 @@ def build_message_bodies(analyse_container_type): return message_dict storage = get_s3_storage() - for bucket_name, pdf_name in storage.get_all_object_names(CONFIG.storage.bucket): + for bucket_name, pdf_name in storage.get_all_object_names(parse_disjunction_string(CONFIG.storage.bucket)): if "pdf" not in pdf_name: continue file_id = pdf_name.split(".")[0]