Pull request #53: Feature/MLOPS-23 pyinfra does not use the value of the storage azurecontainername environment

Merge in RR/pyinfra from feature/MLOPS-23-pyinfra-does-not-use-the-value-of-the-storage_azurecontainername-environment to master

* commit '7a740403bb65db97c8e4cb54de00aac3536b2e4c':
  update
  update test config
  update test config
  add pytests to check if a configured bucket can be found
  add submodule initialization
  load different env vars for the  variable depending on the set

README.md

@@ -7,7 +7,7 @@ The Infrastructure expects to be deployed in the same Pod / local environment as
 A configuration is located in `/config.yaml`. All relevant variables can be configured via exporting environment variables.
 
 | Environment Variable          | Default                        | Description                                                           |
-|-------------------------------|--------------------------------|-----------------------------------------------------------------------|
+| ----------------------------- | ------------------------------ | --------------------------------------------------------------------- |
 | LOGGING_LEVEL_ROOT            | DEBUG                          | Logging level for service logger                                      |
 | PROBING_WEBSERVER_HOST        | "0.0.0.0"                      | Probe webserver address                                               |
 | PROBING_WEBSERVER_PORT        | 8080                           | Probe webserver port                                                  |
@@ -22,11 +22,12 @@ A configuration is located in `/config.yaml`. All relevant variables can be conf
 | DEAD_LETTER_QUEUE             | dead_letter_queue              | Messages that failed to process                                       |
 | ANALYSIS_ENDPOINT             | "http://127.0.0.1:5000"        | Endpoint for analysis container                                       |
 | STORAGE_BACKEND               | s3                             | The type of storage to use {s3, azure}                                |
-| STORAGE_BUCKET                | "pyinfra-test-bucket"          | The bucket / container to pull files specified in queue requests from |
+| STORAGE_BUCKET                | "redaction"                    | The bucket / container to pull files specified in queue requests from |
 | STORAGE_ENDPOINT              | "http://127.0.0.1:9000"        | Endpoint for s3 storage                                               |
 | STORAGE_KEY                   | root                           | User for s3 storage                                                   |
 | STORAGE_SECRET                | password                       | Password for s3 storage                                               |
 | STORAGE_AZURECONNECTIONSTRING | "DefaultEndpointsProtocol=..." | Connection string for Azure storage                                   |
+| STORAGE_AZURECONTAINERNAME    | "redaction"                    | AKS container                                                         |
 
 ## Response Format
 

pyinfra/__init__.py

@@ -0,0 +1,3 @@
+from pyinfra import k8s_probes, queue, storage, config
+
+__all__ = ["k8s_probes", "queue", "storage", "config"]

pyinfra/config.py

@@ -38,7 +38,10 @@ class Config(object):
         self.storage_backend = read_from_environment("STORAGE_BACKEND", "s3")
 
         # The bucket / container to pull files specified in queue requests from
-        self.storage_bucket = read_from_environment("STORAGE_BUCKET_NAME", "redaction")
+        if self.storage_backend == "s3":
+            self.storage_bucket = read_from_environment("STORAGE_BUCKET_NAME", "redaction")
+        else:
+            self.storage_bucket = read_from_environment("STORAGE_AZURECONTAINERNAME", "redaction")
 
         # Endpoint for s3 storage
         self.storage_endpoint = read_from_environment("STORAGE_ENDPOINT", "http://127.0.0.1:9000")

pyinfra/k8s_probes/__init__.py

@@ -0,0 +1,3 @@
+from pyinfra.k8s_probes import startup
+
+__all__ = ["startup"]

pyinfra/queue/__init__.py

@@ -0,0 +1,3 @@
+from pyinfra.queue import queue_manager
+
+__all__ = ["queue_manager"]

pyinfra/storage/__init__.py

@@ -0,0 +1,4 @@
+from pyinfra.storage import adapters, storage
+from pyinfra.storage.storage import get_storage
+
+__all__ = ["adapters", "storage"]

tests/conftest.py

@@ -0,0 +1,19 @@
+import pytest
+from pyinfra.config import get_config, Config
+import os
+
+@pytest.fixture(params=["aws", "azure"])
+def storage_config(request) -> Config:
+    if request.param == "aws":
+        os.environ["STORAGE_BACKEND"] = "s3"
+        os.environ["STORAGE_BUCKET_NAME"] = "pyinfra-test-bucket"
+        os.environ["STORAGE_ENDPOINT"] = "https://s3.amazonaws.com"
+        os.environ["STORAGE_KEY"] = "AKIA4QVP6D4LCDAGYGN2"
+        os.environ["STORAGE_SECRET"] = "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"
+        os.environ["STORAGE_REGION"] = "eu-west-1"
+    else:
+        os.environ["STORAGE_BACKEND"] = "azure"
+        os.environ["STORAGE_AZURECONTAINERNAME"] = "pyinfra-test-bucket"
+        os.environ["STORAGE_AZURECONNECTIONSTRING"] = "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
+    
+    return get_config()

tests/test_storage.py

@@ -0,0 +1,5 @@
+from pyinfra.storage import get_storage
+
+def test_storage(storage_config) -> None:
+    storage = get_storage(storage_config)
+    assert storage.has_bucket(storage_config.storage_bucket)