From 2128f3bea9a7941649436f5d3b9da8322264702c Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Wed, 23 Feb 2022 21:15:44 +0100 Subject: [PATCH] implementation of azure storage handle WIP --- pyinfra/storage/adapters/__init__.py | 0 pyinfra/storage/{ => adapters}/adapter.py | 0 pyinfra/storage/adapters/azure.py | 59 +++++++++++++++++ pyinfra/storage/clients/__init__.py | 0 pyinfra/storage/clients/azure.py | 8 +++ pyinfra/test/storage/adapter_mock.py | 2 +- pyinfra/test/unit_tests/azure_adapter_test.py | 10 +++ pyinfra/test/unit_tests/conftest.py | 6 ++ .../test/unit_tests/storage_adapter_test.py | 3 +- pyinfra/test/unit_tests/storage_test.py | 66 ++++++++++++------- 10 files changed, 126 insertions(+), 28 deletions(-) create mode 100644 pyinfra/storage/adapters/__init__.py rename pyinfra/storage/{ => adapters}/adapter.py (100%) create mode 100644 pyinfra/storage/adapters/azure.py create mode 100644 pyinfra/storage/clients/__init__.py create mode 100644 pyinfra/storage/clients/azure.py create mode 100644 pyinfra/test/unit_tests/azure_adapter_test.py create mode 100644 pyinfra/test/unit_tests/conftest.py diff --git a/pyinfra/storage/adapters/__init__.py b/pyinfra/storage/adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyinfra/storage/adapter.py b/pyinfra/storage/adapters/adapter.py similarity index 100% rename from pyinfra/storage/adapter.py rename to pyinfra/storage/adapters/adapter.py diff --git a/pyinfra/storage/adapters/azure.py b/pyinfra/storage/adapters/azure.py new file mode 100644 index 0000000..3ced009 --- /dev/null +++ b/pyinfra/storage/adapters/azure.py @@ -0,0 +1,59 @@ +import logging +from functools import wraps + +from azure.core.exceptions import ResourceExistsError +from azure.storage.blob import ContainerClient, BlobServiceClient, BlobClient +from retry import retry + +from pyinfra.storage.adapters.adapter import StorageAdapter + + +def _retry(exceptions=Exception): + + def inner(func): + + @retry(exceptions=exceptions, delay=5, jitter=(0, 3), max_delay=60) + @wraps(func) + def inner(*args, **kwargs): + return func(*args, **kwargs) + + return inner + + return inner + + +class AzureStorageAdapter(StorageAdapter): + + def __init__(self, client): + super().__init__(client=client) + self.__client: BlobServiceClient = self._StorageAdapter__client + + def __provide_container_client(self, container_name) -> ContainerClient: + container_client = self.__client.get_container_client(container_name) + return container_client if container_client.exists() else self.__client.create_container(container_name) + + def put(self, bucket_name, object_name, data): + pass + # container_client = self.__provide_container_client(bucket_name) + # blob_client = container_client.get_blob_client(object_name) + # blob_client.upload_blob(data, overwrite=True) + + def get(self, bucket_name, object_name): + pass + # container_client = self.__provide_container_client(bucket_name) + # blob_client = container_client.get_blob_client(object_name) + # blob_data = blob_client.download_blob() + # return blob_data + + def get_all(self, bucket_name): + container_client = self.__provide_container_client(bucket_name) + for blob in container_client.list_blobs(bucket_name): + blob_client = container_client.get_blob_client(blob) + blob_data = blob_client.download_blob() + yield blob_data + + @_retry(ResourceExistsError) + def purge(self, bucket_name): + logging.debug(f"Purging Azure container '{bucket_name}'") + self.__client.delete_container(bucket_name) + self.__provide_container_client(bucket_name) diff --git a/pyinfra/storage/clients/__init__.py b/pyinfra/storage/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyinfra/storage/clients/azure.py b/pyinfra/storage/clients/azure.py new file mode 100644 index 0000000..c8c50da --- /dev/null +++ b/pyinfra/storage/clients/azure.py @@ -0,0 +1,8 @@ +from azure.storage.blob import BlobServiceClient + +from pyinfra.config import CONFIG + + +def get_azure_client(connection_string=None) -> BlobServiceClient: + connection_string = CONFIG.azure_blob_storage.connection_string if connection_string is None else connection_string + return BlobServiceClient.from_connection_string(conn_str=connection_string) diff --git a/pyinfra/test/storage/adapter_mock.py b/pyinfra/test/storage/adapter_mock.py index 34280fe..467dac3 100644 --- a/pyinfra/test/storage/adapter_mock.py +++ b/pyinfra/test/storage/adapter_mock.py @@ -1,4 +1,4 @@ -from pyinfra.storage.adapter import StorageAdapter +from pyinfra.storage.adapters.adapter import StorageAdapter from pyinfra.test.storage.client_mock import StorageClientMock diff --git a/pyinfra/test/unit_tests/azure_adapter_test.py b/pyinfra/test/unit_tests/azure_adapter_test.py new file mode 100644 index 0000000..d758668 --- /dev/null +++ b/pyinfra/test/unit_tests/azure_adapter_test.py @@ -0,0 +1,10 @@ +import pytest + +from pyinfra.storage.adapters.azure import AzureStorageAdapter +from pyinfra.test.storage.client_mock import StorageClientMock + + +@pytest.fixture +def adapter(): + adapter = AzureStorageAdapter(StorageClientMock()) + return adapter diff --git a/pyinfra/test/unit_tests/conftest.py b/pyinfra/test/unit_tests/conftest.py new file mode 100644 index 0000000..e54ce26 --- /dev/null +++ b/pyinfra/test/unit_tests/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture +def bucket_name(): + return "test-bucket" \ No newline at end of file diff --git a/pyinfra/test/unit_tests/storage_adapter_test.py b/pyinfra/test/unit_tests/storage_adapter_test.py index d0aa86c..8b13789 100644 --- a/pyinfra/test/unit_tests/storage_adapter_test.py +++ b/pyinfra/test/unit_tests/storage_adapter_test.py @@ -1,2 +1 @@ -from pyinfra.storage.adapter import StorageAdapter -from pyinfra.test.storage.client_mock import StorageClientMock + diff --git a/pyinfra/test/unit_tests/storage_test.py b/pyinfra/test/unit_tests/storage_test.py index 3245e32..d8ec641 100644 --- a/pyinfra/test/unit_tests/storage_test.py +++ b/pyinfra/test/unit_tests/storage_test.py @@ -2,38 +2,54 @@ import tempfile import pytest +from pyinfra.storage.adapters.azure import AzureStorageAdapter +from pyinfra.storage.clients.azure import get_azure_client from pyinfra.storage.storage import Storage from pyinfra.test.storage.adapter_mock import StorageAdapterMock from pyinfra.test.storage.client_mock import StorageClientMock -@pytest.fixture -def storage(): - storage = Storage(adapter=StorageAdapterMock(StorageClientMock())) - return storage +class StorageTester: + + def __init__(self, adapter): + self.bucket_name = "pyinfra-test-bucket" + self.storage = Storage(adapter=adapter) + + def test_purging_bucket_yields_empty_bucket(self): + self.storage.purge(self.bucket_name) + data_received = self.storage.get_all(self.bucket_name) + assert not {*data_received} + + def test_getting_object_put_in_bucket_is_object(self): + with tempfile.SpooledTemporaryFile() as f: + self.storage.put(self.bucket_name, "file", f) + data_received = self.storage.get(self.bucket_name, "file") + assert f == data_received + + def test_getting_objects_put_in_bucket_are_objects(self): + with tempfile.SpooledTemporaryFile() as f1, tempfile.SpooledTemporaryFile() as f2: + self.storage.put(self.bucket_name, "file1", f1) + self.storage.put(self.bucket_name, "file2", f2) + data_received = self.storage.get_all(self.bucket_name) + assert {f1, f2} == {*data_received} + + +def get_client(client_name): + if client_name == "mock": + return StorageAdapterMock(StorageClientMock()) + if client_name == "azure": + return AzureStorageAdapter(get_azure_client()) @pytest.fixture -def bucket_name(): - return "test-bucket" +def storage_tester(client_name): + client = get_client(client_name) + return StorageTester(client) -def test_purging_bucket_yields_empty_bucket(storage, bucket_name): - storage.purge(bucket_name) - data_received = storage.get_all(bucket_name) - assert not {*data_received} - - -def test_getting_object_put_in_bucket_is_object(storage, bucket_name): - with tempfile.SpooledTemporaryFile() as f: - storage.put(bucket_name, "file", f) - data_received = storage.get(bucket_name, "file") - assert f == data_received - - -def test_getting_objects_put_in_bucket_are_objects(storage, bucket_name): - with tempfile.SpooledTemporaryFile() as f1, tempfile.SpooledTemporaryFile() as f2: - storage.put(bucket_name, "file1", f1) - storage.put(bucket_name, "file2", f2) - data_received = storage.get_all(bucket_name) - assert {f1, f2} == {*data_received} +@pytest.mark.parametrize('client_name', ["mock", "azure"]) +def test_storages(storage_tester): + storage_tester.test_purging_bucket_yields_empty_bucket() + storage_tester.test_getting_objects_put_in_bucket_are_objects() + # storage_tester.test_getting_object_put_in_bucket_is_object() + assert True