Pull request #27: Add parser for bucket/container env vars

Merge in RR/pyinfra from kbudisantoso/configyaml-1650538128334 to master

Squashed commit of the following:

commit 6103b7720315aaef3d98aea8f3c817477bbf500b
Merge: 69ac65a 3b91185
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date:   Thu Apr 21 14:45:10 2022 +0200

    Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334

commit 69ac65ae1bd4095c797112c6f9530f0b1705277e
Merge: 9a1cd07 a00ceae
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date:   Thu Apr 21 14:37:34 2022 +0200

    Merge remote-tracking branch 'origin' into kbudisantoso/configyaml-1650538128334

commit 9a1cd07c09e5ee2618f2c1a3c27b69c67b1eaeb0
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date:   Thu Apr 21 14:35:49 2022 +0200

    test done

commit e7127e8af937fe067f1f92eb688187ebbe609478
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date:   Thu Apr 21 14:32:25 2022 +0200

    test

commit 262957e33d19dbafb3f10b5a32c438460b966a88
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date:   Thu Apr 21 14:16:33 2022 +0200

    add parser for env var storage_buckets/containers

commit 3535002b4aac9f297bdbe112b04f537cef25f5c2
Author: Kresnadi Budisantoso <kresnadi.budisantoso@iqser.com>
Date:   Thu Apr 21 12:48:52 2022 +0200

    config.yaml online editiert mit Bitbucket

config.yaml

@@ -23,7 +23,7 @@ rabbitmq:
 
 storage:
   backend: $STORAGE_BACKEND|s3  # The type of storage to use {s3, azure}
-  bucket: $STORAGE_BUCKET|"pyinfra-test-bucket"  # The bucket / container to pull files specified in queue requests from
+  bucket: "STORAGE_BUCKET_NAME|STORAGE_AZURECONTAINERNAME|pyinfra-test-bucket"  # The bucket / container to pull files specified in queue requests from
 
   s3:
     endpoint: $STORAGE_ENDPOINT|"http://127.0.0.1:9000"

pyinfra/config.py

@@ -1,7 +1,9 @@
 """Implements a config object with dot-indexing syntax."""
-
+import os
+from itertools import chain
 
 from envyaml import EnvYAML
+from funcy import first, juxt, butlast, last
 
 from pyinfra.locations import CONFIG_FILE
 
@@ -38,3 +40,15 @@ class Config:
 
 
 CONFIG = Config(CONFIG_FILE)
+
+
+def parse_disjunction_string(disjunction_string):
+    def try_parse_env_var(disjunction_string):
+        try:
+            return os.environ[disjunction_string]
+        except KeyError:
+            return None
+
+    options = disjunction_string.split("|")
+    identifiers, fallback_value = juxt(butlast, last)(options)
+    return first(chain(filter(try_parse_env_var, identifiers), [fallback_value]))

pyinfra/visitor.py

@@ -5,7 +5,7 @@ import logging
 from operator import itemgetter
 from typing import Callable
 
-from pyinfra.config import CONFIG
+from pyinfra.config import CONFIG, parse_disjunction_string
 from pyinfra.exceptions import DataLoadingFailure
 from pyinfra.storage.storage import Storage
 
@@ -23,11 +23,12 @@ def get_response_object_name(body):
 
 
 def get_object_descriptor(body):
-    return {"bucket_name": CONFIG.storage.bucket, "object_name": get_object_name(body)}
+    return {"bucket_name": parse_disjunction_string(CONFIG.storage.bucket), "object_name": get_object_name(body)}
 
 
 def get_response_object_descriptor(body):
-    return {"bucket_name": CONFIG.storage.bucket, "object_name": get_response_object_name(body)}
+    return {"bucket_name": parse_disjunction_string(CONFIG.storage.bucket),
+            "object_name": get_response_object_name(body)}
 
 
 class ResponseStrategy(abc.ABC):

requirements.txt

@@ -11,3 +11,4 @@ testcontainers==3.4.2
 docker-compose==1.29.2
 tqdm==4.62.3
 pytest~=7.0.1
+funcy==1.17

scripts/manage_minio.py

@@ -5,7 +5,7 @@ from pathlib import Path
 
 from tqdm import tqdm
 
-from pyinfra.config import CONFIG
+from pyinfra.config import CONFIG, parse_disjunction_string
 from pyinfra.storage.storages import get_s3_storage
 
 
@@ -51,7 +51,7 @@ def add_file_compressed(storage, bucket_name, dossier_id, path) -> None:
 if __name__ == "__main__":
 
     storage = get_s3_storage()
-    bucket_name = CONFIG.storage.bucket
+    bucket_name = parse_disjunction_string(CONFIG.storage.bucket)
 
     if not storage.has_bucket(bucket_name):
         storage.make_bucket(bucket_name)

scripts/mock_client.py

@@ -3,7 +3,7 @@ import json
 
 import pika
 
-from pyinfra.config import CONFIG
+from pyinfra.config import CONFIG, parse_disjunction_string
 from pyinfra.storage.storages import get_s3_storage
 
 
@@ -53,7 +53,7 @@ def build_message_bodies(analyse_container_type):
         return message_dict
 
     storage = get_s3_storage()
-    for bucket_name, pdf_name in storage.get_all_object_names(CONFIG.storage.bucket):
+    for bucket_name, pdf_name in storage.get_all_object_names(parse_disjunction_string(CONFIG.storage.bucket)):
         if "pdf" not in pdf_name:
             continue
         file_id = pdf_name.split(".")[0]