implementation of azure storage handle WIP

pyinfra/storage/adapters/azure.py

@@ -0,0 +1,59 @@
+import logging
+from functools import wraps
+
+from azure.core.exceptions import ResourceExistsError
+from azure.storage.blob import ContainerClient, BlobServiceClient, BlobClient
+from retry import retry
+
+from pyinfra.storage.adapters.adapter import StorageAdapter
+
+
+def _retry(exceptions=Exception):
+    
+    def inner(func):
+
+        @retry(exceptions=exceptions, delay=5, jitter=(0, 3), max_delay=60)
+        @wraps(func)
+        def inner(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return inner
+
+    return inner
+
+
+class AzureStorageAdapter(StorageAdapter):
+
+    def __init__(self, client):
+        super().__init__(client=client)
+        self.__client: BlobServiceClient = self._StorageAdapter__client
+
+    def __provide_container_client(self, container_name) -> ContainerClient:
+        container_client = self.__client.get_container_client(container_name)
+        return container_client if container_client.exists() else self.__client.create_container(container_name)
+
+    def put(self, bucket_name, object_name, data):
+        pass
+        # container_client = self.__provide_container_client(bucket_name)
+        # blob_client = container_client.get_blob_client(object_name)
+        # blob_client.upload_blob(data, overwrite=True)
+
+    def get(self, bucket_name, object_name):
+        pass
+        # container_client = self.__provide_container_client(bucket_name)
+        # blob_client = container_client.get_blob_client(object_name)
+        # blob_data = blob_client.download_blob()
+        # return blob_data
+
+    def get_all(self, bucket_name):
+        container_client = self.__provide_container_client(bucket_name)
+        for blob in container_client.list_blobs(bucket_name):
+            blob_client = container_client.get_blob_client(blob)
+            blob_data = blob_client.download_blob()
+            yield blob_data
+
+    @_retry(ResourceExistsError)
+    def purge(self, bucket_name):
+        logging.debug(f"Purging Azure container '{bucket_name}'")
+        self.__client.delete_container(bucket_name)
+        self.__provide_container_client(bucket_name)

pyinfra/storage/clients/azure.py

@@ -0,0 +1,8 @@
+from azure.storage.blob import BlobServiceClient
+
+from pyinfra.config import CONFIG
+
+
+def get_azure_client(connection_string=None) -> BlobServiceClient:
+    connection_string = CONFIG.azure_blob_storage.connection_string if connection_string is None else connection_string
+    return BlobServiceClient.from_connection_string(conn_str=connection_string)

pyinfra/test/storage/adapter_mock.py

@@ -1,4 +1,4 @@
-from pyinfra.storage.adapter import StorageAdapter
+from pyinfra.storage.adapters.adapter import StorageAdapter
 from pyinfra.test.storage.client_mock import StorageClientMock
 
 

pyinfra/test/unit_tests/azure_adapter_test.py

@@ -0,0 +1,10 @@
+import pytest
+
+from pyinfra.storage.adapters.azure import AzureStorageAdapter
+from pyinfra.test.storage.client_mock import StorageClientMock
+
+
+@pytest.fixture
+def adapter():
+    adapter = AzureStorageAdapter(StorageClientMock())
+    return adapter

pyinfra/test/unit_tests/conftest.py

@@ -0,0 +1,6 @@
+import pytest
+
+
+@pytest.fixture
+def bucket_name():
+    return "test-bucket"

pyinfra/test/unit_tests/storage_adapter_test.py

@@ -1,2 +1 @@
-from pyinfra.storage.adapter import StorageAdapter
-from pyinfra.test.storage.client_mock import StorageClientMock
+

pyinfra/test/unit_tests/storage_test.py

@@ -2,38 +2,54 @@ import tempfile
 
 import pytest
 
+from pyinfra.storage.adapters.azure import AzureStorageAdapter
+from pyinfra.storage.clients.azure import get_azure_client
 from pyinfra.storage.storage import Storage
 from pyinfra.test.storage.adapter_mock import StorageAdapterMock
 from pyinfra.test.storage.client_mock import StorageClientMock
 
 
-@pytest.fixture
-def storage():
-    storage = Storage(adapter=StorageAdapterMock(StorageClientMock()))
-    return storage
+class StorageTester:
+
+    def __init__(self, adapter):
+        self.bucket_name = "pyinfra-test-bucket"
+        self.storage = Storage(adapter=adapter)
+
+    def test_purging_bucket_yields_empty_bucket(self):
+        self.storage.purge(self.bucket_name)
+        data_received = self.storage.get_all(self.bucket_name)
+        assert not {*data_received}
+
+    def test_getting_object_put_in_bucket_is_object(self):
+        with tempfile.SpooledTemporaryFile() as f:
+            self.storage.put(self.bucket_name, "file", f)
+            data_received = self.storage.get(self.bucket_name, "file")
+            assert f == data_received
+
+    def test_getting_objects_put_in_bucket_are_objects(self):
+        with tempfile.SpooledTemporaryFile() as f1, tempfile.SpooledTemporaryFile() as f2:
+            self.storage.put(self.bucket_name, "file1", f1)
+            self.storage.put(self.bucket_name, "file2", f2)
+            data_received = self.storage.get_all(self.bucket_name)
+            assert {f1, f2} == {*data_received}
+
+
+def get_client(client_name):
+    if client_name == "mock":
+        return StorageAdapterMock(StorageClientMock())
+    if client_name == "azure":
+        return AzureStorageAdapter(get_azure_client())
 
 
 @pytest.fixture
-def bucket_name():
-    return "test-bucket"
+def storage_tester(client_name):
+    client = get_client(client_name)
+    return StorageTester(client)
 
 
-def test_purging_bucket_yields_empty_bucket(storage, bucket_name):
-    storage.purge(bucket_name)
-    data_received = storage.get_all(bucket_name)
-    assert not {*data_received}
-
-
-def test_getting_object_put_in_bucket_is_object(storage, bucket_name):
-    with tempfile.SpooledTemporaryFile() as f:
-        storage.put(bucket_name, "file", f)
-        data_received = storage.get(bucket_name, "file")
-        assert f == data_received
-
-
-def test_getting_objects_put_in_bucket_are_objects(storage, bucket_name):
-    with tempfile.SpooledTemporaryFile() as f1, tempfile.SpooledTemporaryFile() as f2:
-        storage.put(bucket_name, "file1", f1)
-        storage.put(bucket_name, "file2", f2)
-        data_received = storage.get_all(bucket_name)
-        assert {f1, f2} == {*data_received}
+@pytest.mark.parametrize('client_name', ["mock", "azure"])
+def test_storages(storage_tester):
+    storage_tester.test_purging_bucket_yields_empty_bucket()
+    storage_tester.test_getting_objects_put_in_bucket_are_objects()
+    # storage_tester.test_getting_object_put_in_bucket_is_object()
+    assert True