defined test sections in config for enpoints of storages

config.yaml

@@ -36,3 +36,21 @@ probing_webserver:
   host: $PROBING_WEBSERVER_HOST|"0.0.0.0"  # Probe webserver address
   port: $PROBING_WEBSERVER_PORT|8080  # Probe webserver port
   mode: $PROBING_WEBSERVER_MODE|production  # webserver mode: {development, production}
+
+
+test:
+  minio:
+    s3:
+      endpoint: "http://127.0.0.1:9000"
+      access_key: root
+      secret_key: password
+
+  aws:
+    s3:
+      endpoint: https://s3.amazonaws.com
+      access_key: AKIA4QVP6D4LCDAGYGN2
+      secret_key: 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
+
+  azure:
+    azure_blob_storage:
+      connection_string: "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"

pyinfra/storage/adapters/adapter.py

@@ -26,5 +26,5 @@ class StorageAdapter(ABC):
         pass
 
     @abstractmethod
-    def purge(self, bucket_name):
+    def clear_bucket(self, bucket_name):
         pass

pyinfra/storage/adapters/azure.py

@@ -51,6 +51,7 @@ class AzureStorageAdapter(StorageAdapter):
         blob_client.upload_blob(data, overwrite=True)
 
     def get(self, bucket_name, object_name):
+        logger.debug(f"Downloading '{object_name}'...")
         container_client = self.__provide_container_client(bucket_name)
         blob_client = container_client.get_blob_client(object_name)
         blob_data = blob_client.download_blob()
@@ -67,8 +68,8 @@ class AzureStorageAdapter(StorageAdapter):
             data = blob_data.readall()
             yield data
 
-    def purge(self, bucket_name):
-        logger.debug(f"Purging Azure container '{bucket_name}'...")
+    def clear_bucket(self, bucket_name):
+        logger.debug(f"Clearing Azure container '{bucket_name}'...")
         container_client = self.__client.get_container_client(bucket_name)
         blobs = container_client.list_blobs()
         container_client.delete_blobs(*blobs)

pyinfra/storage/adapters/s3.py

@@ -1,9 +1,11 @@
 import io
+import logging
 
 from minio import Minio
 
 from pyinfra.storage.adapters.adapter import StorageAdapter
 
+logger = logging.getLogger(__name__)
 
 class S3StorageAdapter(StorageAdapter):
 
@@ -19,10 +21,12 @@ class S3StorageAdapter(StorageAdapter):
         return self.__client.bucket_exists(bucket_name)
 
     def put(self, bucket_name, object_name, data):
+        logger.debug(f"Uploading '{object_name}'...")
         data = io.BytesIO(data)
         self.__client.put_object(bucket_name, object_name, data, length=data.getbuffer().nbytes)
 
     def get(self, bucket_name, object_name):
+        logger.debug(f"Downloading '{object_name}'...")
         response = None
 
         try:
@@ -35,9 +39,11 @@ class S3StorageAdapter(StorageAdapter):
 
     def get_all(self, bucket_name):
         for obj in self.__client.list_objects(bucket_name):
+            logger.debug(f"Downloading '{obj.object_name}'...")
             yield self.get(bucket_name, obj.object_name)
 
-    def purge(self, bucket_name):
+    def clear_bucket(self, bucket_name):
+        logger.debug(f"Clearing S3 bucket '{bucket_name}'...")
         objects = self.__client.list_objects(bucket_name)
         for obj in objects:
             self.__client.remove_object(bucket_name, obj.object_name)

pyinfra/storage/clients/azure.py

@@ -3,6 +3,7 @@ from azure.storage.blob import BlobServiceClient
 from pyinfra.config import CONFIG
 
 
-def get_azure_client(connection_string=None) -> BlobServiceClient:
-    connection_string = CONFIG.azure_blob_storage.connection_string if connection_string is None else connection_string
-    return BlobServiceClient.from_connection_string(conn_str=connection_string)
+def get_azure_client(config=None) -> BlobServiceClient:
+    if not config:
+        config = CONFIG
+    return BlobServiceClient.from_connection_string(conn_str=config.azure_blob_storage.connection_string)

pyinfra/storage/clients/s3.py

@@ -17,6 +17,11 @@ def parse_endpoint(endpoint):
     return {"secure": match.group("protocol") == "https", "endpoint": match.group("address")}
 
 
-def get_s3_client(endpoint=None) -> Minio:
-    endpoint = CONFIG.s3.endpoint if endpoint is None else endpoint
-    return Minio(**parse_endpoint(endpoint), access_key=CONFIG.s3.access_key, secret_key=CONFIG.s3.secret_key)
+def get_s3_client(config=None) -> Minio:
+
+    if not config:
+        config = CONFIG
+
+    endpoint = config.s3.endpoint
+
+    return Minio(**parse_endpoint(endpoint), access_key=config.s3.access_key, secret_key=config.s3.secret_key)

pyinfra/storage/storage.py

@@ -17,8 +17,8 @@ class Storage:
     def get_all(self, bucket_name):
         return self.__adapter.get_all(bucket_name)
 
-    def purge(self, bucket_name):
-        return self.__adapter.purge(bucket_name)
+    def clear_bucket(self, bucket_name):
+        return self.__adapter.clear_bucket(bucket_name)
 
 
 

pyinfra/test/storage/adapter_mock.py

@@ -23,5 +23,5 @@ class StorageAdapterMock(StorageAdapter):
     def get_all(self, bucket_name):
         return self.__client.get_all(bucket_name)
 
-    def purge(self, bucket_name):
-        return self.__client.purge(bucket_name)
+    def clear_bucket(self, bucket_name):
+        return self.__client.clear_bucket(bucket_name)

pyinfra/test/storage/client_mock.py

@@ -17,5 +17,5 @@ class StorageClientMock:
     def get_all(self, bucket_name):
         return self.__data[bucket_name].values()
 
-    def purge(self, bucket_name):
+    def clear_bucket(self, bucket_name):
         self.__data[bucket_name] = {}

pyinfra/test/unit_tests/storage_test.py

@@ -2,6 +2,7 @@ import logging
 
 import pytest
 
+from pyinfra.config import CONFIG
 from pyinfra.exceptions import UnknownClient
 from pyinfra.storage.adapters.azure import AzureStorageAdapter
 from pyinfra.storage.adapters.s3 import S3StorageAdapter
@@ -14,8 +15,8 @@ from pyinfra.test.storage.client_mock import StorageClientMock
 
 @pytest.mark.parametrize("client_name", ["mock", "azure", "s3"])
 class TestStorage:
-    def test_purging_bucket_yields_empty_bucket(self, storage, bucket_name):
-        storage.purge(bucket_name)
+    def test_clearing_bucket_yields_empty_bucket(self, storage, bucket_name):
+        storage.clear_bucket(bucket_name)
         data_received = storage.get_all(bucket_name)
         assert not {*data_received}
 
@@ -34,14 +35,13 @@ class TestStorage:
         storage.make_bucket(bucket_name)
         assert storage.bucket_exists(bucket_name)
 
-
 def get_adapter(client_name):
     if client_name == "mock":
         return StorageAdapterMock(StorageClientMock())
     if client_name == "azure":
-        return AzureStorageAdapter(get_azure_client())
+        return AzureStorageAdapter(get_azure_client(CONFIG.test.azure))
     if client_name == "s3":
-        return S3StorageAdapter(get_s3_client())
+        return S3StorageAdapter(get_s3_client(CONFIG.test.minio))
     else:
         raise UnknownClient(client_name)
 
@@ -56,6 +56,6 @@ def adapter(client_name):
 def storage(client_name, bucket_name):
     storage = Storage(get_adapter(client_name))
     storage.make_bucket(bucket_name)
-    storage.purge(bucket_name)
+    storage.clear_bucket(bucket_name)
     logging.info(client_name)
     return storage