From bb5b73e189b161d8194bdd424e17fb3c187f52b2 Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 10:29:57 +0200 Subject: [PATCH 1/6] load different env vars for the variable depending on the set --- pyinfra/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyinfra/config.py b/pyinfra/config.py index a08f5ab..3ebcd4f 100644 --- a/pyinfra/config.py +++ b/pyinfra/config.py @@ -38,7 +38,10 @@ class Config(object): self.storage_backend = read_from_environment("STORAGE_BACKEND", "s3") # The bucket / container to pull files specified in queue requests from - self.storage_bucket = read_from_environment("STORAGE_BUCKET_NAME", "redaction") + if self.storage_backend == "s3": + self.storage_bucket = read_from_environment("STORAGE_BUCKET_NAME", "redaction") + else: + self.storage_bucket = read_from_environment("STORAGE_AZURECONTAINERNAME", "redaction") # Endpoint for s3 storage self.storage_endpoint = read_from_environment("STORAGE_ENDPOINT", "http://127.0.0.1:9000") From 5b948fdcc5b079fd309a1fe091ae822abe774e1b Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 11:01:29 +0200 Subject: [PATCH 2/6] add submodule initialization --- pyinfra/__init__.py | 3 +++ pyinfra/k8s_probes/__init__.py | 3 +++ pyinfra/queue/__init__.py | 3 +++ pyinfra/storage/__init__.py | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/pyinfra/__init__.py b/pyinfra/__init__.py index e69de29..9d92b43 100644 --- a/pyinfra/__init__.py +++ b/pyinfra/__init__.py @@ -0,0 +1,3 @@ +from pyinfra import k8s_probes, queue, storage, config + +__all__ = ["k8s_probes", "queue", "storage", "config"] \ No newline at end of file diff --git a/pyinfra/k8s_probes/__init__.py b/pyinfra/k8s_probes/__init__.py index e69de29..ecc33ff 100644 --- a/pyinfra/k8s_probes/__init__.py +++ b/pyinfra/k8s_probes/__init__.py @@ -0,0 +1,3 @@ +from pyinfra.k8s_probes import startup + +__all__ = ["startup"] \ No newline at end of file diff --git a/pyinfra/queue/__init__.py b/pyinfra/queue/__init__.py index e69de29..c980a1e 100644 --- a/pyinfra/queue/__init__.py +++ b/pyinfra/queue/__init__.py @@ -0,0 +1,3 @@ +from pyinfra.queue import queue_manager + +__all__ = ["queue_manager"] \ No newline at end of file diff --git a/pyinfra/storage/__init__.py b/pyinfra/storage/__init__.py index e69de29..f01d6da 100644 --- a/pyinfra/storage/__init__.py +++ b/pyinfra/storage/__init__.py @@ -0,0 +1,4 @@ +from pyinfra.storage import adapters, storage +from pyinfra.storage.storage import get_storage + +__all__ = ["adapters", "storage"] \ No newline at end of file From 843d91c61ad3967a12290aca370658f2f3dd328e Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 11:26:58 +0200 Subject: [PATCH 3/6] add pytests to check if a configured bucket can be found --- test/__init__.py | 0 tests/conftest.py | 15 +++++++++++++++ tests/test_storage.py | 5 +++++ 3 files changed, 20 insertions(+) delete mode 100644 test/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_storage.py diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7c80754 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,15 @@ +from distutils.command.config import config +import pytest +from pyinfra.config import get_config +import os + +@pytest.fixture(params=) +def storage_config(request): + if request.param == "s3": + os.environ["STORAGE_BACKEND"] = "s3" + os.environ["STORAGE_BUCKET_NAME"] = "pyinfra-test-bucket" + else: + os.environ["STORAGE_BACKEND"] = "aks" + os.environ["STORAGE_AZURECONTAINERNAME"] = "pyinfra-test-bucket" + + return get_config() diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..830f9df --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,5 @@ +from pyinfra.storage import get_storage + +def test_storage(storage_config) -> None: + storage = get_storage(storage_config) + assert storage.has_bucket(storage_config.storage_bucket) From 1891519e196188e054c1ffe90d8244d9d8ae122f Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 13:17:31 +0200 Subject: [PATCH 4/6] update test config --- tests/conftest.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7c80754..158fbf5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,9 @@ -from distutils.command.config import config import pytest -from pyinfra.config import get_config +from pyinfra.config import get_config, Config import os -@pytest.fixture(params=) -def storage_config(request): +@pytest.fixture(params=["s3", "aks"]) +def storage_config(request) -> Config: if request.param == "s3": os.environ["STORAGE_BACKEND"] = "s3" os.environ["STORAGE_BUCKET_NAME"] = "pyinfra-test-bucket" From b2cd529519e59117065049af4f95b06025652a24 Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 13:53:53 +0200 Subject: [PATCH 5/6] update test config --- tests/conftest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 158fbf5..247b7ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,13 +2,18 @@ import pytest from pyinfra.config import get_config, Config import os -@pytest.fixture(params=["s3", "aks"]) +@pytest.fixture(params=["aws", "azure"]) def storage_config(request) -> Config: - if request.param == "s3": + if request.param == "aws": os.environ["STORAGE_BACKEND"] = "s3" os.environ["STORAGE_BUCKET_NAME"] = "pyinfra-test-bucket" + os.environ["STORAGE_ENDPOINT"] = "https://s3.amazonaws.com" + os.environ["STORAGE_KEY"] = "AKIA4QVP6D4LCDAGYGN2" + os.environ["STORAGE_SECRET"] = "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" + os.environ["STORAGE_REGION"] = "eu-west-1" else: - os.environ["STORAGE_BACKEND"] = "aks" + os.environ["STORAGE_BACKEND"] = "azure" os.environ["STORAGE_AZURECONTAINERNAME"] = "pyinfra-test-bucket" + os.environ["STORAGE_AZURECONNECTIONSTRING"] = "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net" return get_config() From 7a740403bb65db97c8e4cb54de00aac3536b2e4c Mon Sep 17 00:00:00 2001 From: Francisco Schulz Date: Thu, 13 Oct 2022 14:04:00 +0200 Subject: [PATCH 6/6] update --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33f8e90..b83e886 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The Infrastructure expects to be deployed in the same Pod / local environment as A configuration is located in `/config.yaml`. All relevant variables can be configured via exporting environment variables. | Environment Variable | Default | Description | -|-------------------------------|--------------------------------|-----------------------------------------------------------------------| +| ----------------------------- | ------------------------------ | --------------------------------------------------------------------- | | LOGGING_LEVEL_ROOT | DEBUG | Logging level for service logger | | PROBING_WEBSERVER_HOST | "0.0.0.0" | Probe webserver address | | PROBING_WEBSERVER_PORT | 8080 | Probe webserver port | @@ -22,11 +22,12 @@ A configuration is located in `/config.yaml`. All relevant variables can be conf | DEAD_LETTER_QUEUE | dead_letter_queue | Messages that failed to process | | ANALYSIS_ENDPOINT | "http://127.0.0.1:5000" | Endpoint for analysis container | | STORAGE_BACKEND | s3 | The type of storage to use {s3, azure} | -| STORAGE_BUCKET | "pyinfra-test-bucket" | The bucket / container to pull files specified in queue requests from | +| STORAGE_BUCKET | "redaction" | The bucket / container to pull files specified in queue requests from | | STORAGE_ENDPOINT | "http://127.0.0.1:9000" | Endpoint for s3 storage | | STORAGE_KEY | root | User for s3 storage | | STORAGE_SECRET | password | Password for s3 storage | | STORAGE_AZURECONNECTIONSTRING | "DefaultEndpointsProtocol=..." | Connection string for Azure storage | +| STORAGE_AZURECONTAINERNAME | "redaction" | AKS container | ## Response Format