diff --git a/config.yaml b/config.yaml index 052d4c8..6c21fad 100755 --- a/config.yaml +++ b/config.yaml @@ -36,3 +36,21 @@ probing_webserver: host: $PROBING_WEBSERVER_HOST|"0.0.0.0" # Probe webserver address port: $PROBING_WEBSERVER_PORT|8080 # Probe webserver port mode: $PROBING_WEBSERVER_MODE|production # webserver mode: {development, production} + + +test: + minio: + s3: + endpoint: "http://127.0.0.1:9000" + access_key: root + secret_key: password + + aws: + s3: + endpoint: https://s3.amazonaws.com + access_key: AKIA4QVP6D4LCDAGYGN2 + secret_key: 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 + + azure: + azure_blob_storage: + connection_string: "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net" diff --git a/pyinfra/storage/adapters/adapter.py b/pyinfra/storage/adapters/adapter.py index 7d86631..8a7b129 100644 --- a/pyinfra/storage/adapters/adapter.py +++ b/pyinfra/storage/adapters/adapter.py @@ -26,5 +26,5 @@ class StorageAdapter(ABC): pass @abstractmethod - def purge(self, bucket_name): + def clear_bucket(self, bucket_name): pass diff --git a/pyinfra/storage/adapters/azure.py b/pyinfra/storage/adapters/azure.py index 8092834..1162e51 100644 --- a/pyinfra/storage/adapters/azure.py +++ b/pyinfra/storage/adapters/azure.py @@ -51,6 +51,7 @@ class AzureStorageAdapter(StorageAdapter): blob_client.upload_blob(data, overwrite=True) def get(self, bucket_name, object_name): + logger.debug(f"Downloading '{object_name}'...") container_client = self.__provide_container_client(bucket_name) blob_client = container_client.get_blob_client(object_name) blob_data = blob_client.download_blob() @@ -67,8 +68,8 @@ class AzureStorageAdapter(StorageAdapter): data = blob_data.readall() yield data - def purge(self, bucket_name): - logger.debug(f"Purging Azure container '{bucket_name}'...") + def clear_bucket(self, bucket_name): + logger.debug(f"Clearing Azure container '{bucket_name}'...") container_client = self.__client.get_container_client(bucket_name) blobs = container_client.list_blobs() container_client.delete_blobs(*blobs) diff --git a/pyinfra/storage/adapters/s3.py b/pyinfra/storage/adapters/s3.py index 8faf336..bbf15bf 100644 --- a/pyinfra/storage/adapters/s3.py +++ b/pyinfra/storage/adapters/s3.py @@ -1,9 +1,11 @@ import io +import logging from minio import Minio from pyinfra.storage.adapters.adapter import StorageAdapter +logger = logging.getLogger(__name__) class S3StorageAdapter(StorageAdapter): @@ -19,10 +21,12 @@ class S3StorageAdapter(StorageAdapter): return self.__client.bucket_exists(bucket_name) def put(self, bucket_name, object_name, data): + logger.debug(f"Uploading '{object_name}'...") data = io.BytesIO(data) self.__client.put_object(bucket_name, object_name, data, length=data.getbuffer().nbytes) def get(self, bucket_name, object_name): + logger.debug(f"Downloading '{object_name}'...") response = None try: @@ -35,9 +39,11 @@ class S3StorageAdapter(StorageAdapter): def get_all(self, bucket_name): for obj in self.__client.list_objects(bucket_name): + logger.debug(f"Downloading '{obj.object_name}'...") yield self.get(bucket_name, obj.object_name) - def purge(self, bucket_name): + def clear_bucket(self, bucket_name): + logger.debug(f"Clearing S3 bucket '{bucket_name}'...") objects = self.__client.list_objects(bucket_name) for obj in objects: self.__client.remove_object(bucket_name, obj.object_name) diff --git a/pyinfra/storage/clients/azure.py b/pyinfra/storage/clients/azure.py index c8c50da..526465a 100644 --- a/pyinfra/storage/clients/azure.py +++ b/pyinfra/storage/clients/azure.py @@ -3,6 +3,7 @@ from azure.storage.blob import BlobServiceClient from pyinfra.config import CONFIG -def get_azure_client(connection_string=None) -> BlobServiceClient: - connection_string = CONFIG.azure_blob_storage.connection_string if connection_string is None else connection_string - return BlobServiceClient.from_connection_string(conn_str=connection_string) +def get_azure_client(config=None) -> BlobServiceClient: + if not config: + config = CONFIG + return BlobServiceClient.from_connection_string(conn_str=config.azure_blob_storage.connection_string) diff --git a/pyinfra/storage/clients/s3.py b/pyinfra/storage/clients/s3.py index 10abdd3..6f1f691 100644 --- a/pyinfra/storage/clients/s3.py +++ b/pyinfra/storage/clients/s3.py @@ -17,6 +17,11 @@ def parse_endpoint(endpoint): return {"secure": match.group("protocol") == "https", "endpoint": match.group("address")} -def get_s3_client(endpoint=None) -> Minio: - endpoint = CONFIG.s3.endpoint if endpoint is None else endpoint - return Minio(**parse_endpoint(endpoint), access_key=CONFIG.s3.access_key, secret_key=CONFIG.s3.secret_key) +def get_s3_client(config=None) -> Minio: + + if not config: + config = CONFIG + + endpoint = config.s3.endpoint + + return Minio(**parse_endpoint(endpoint), access_key=config.s3.access_key, secret_key=config.s3.secret_key) diff --git a/pyinfra/storage/storage.py b/pyinfra/storage/storage.py index 0177dc9..b363aa3 100644 --- a/pyinfra/storage/storage.py +++ b/pyinfra/storage/storage.py @@ -17,8 +17,8 @@ class Storage: def get_all(self, bucket_name): return self.__adapter.get_all(bucket_name) - def purge(self, bucket_name): - return self.__adapter.purge(bucket_name) + def clear_bucket(self, bucket_name): + return self.__adapter.clear_bucket(bucket_name) diff --git a/pyinfra/test/storage/adapter_mock.py b/pyinfra/test/storage/adapter_mock.py index 19e4720..4406c5a 100644 --- a/pyinfra/test/storage/adapter_mock.py +++ b/pyinfra/test/storage/adapter_mock.py @@ -23,5 +23,5 @@ class StorageAdapterMock(StorageAdapter): def get_all(self, bucket_name): return self.__client.get_all(bucket_name) - def purge(self, bucket_name): - return self.__client.purge(bucket_name) + def clear_bucket(self, bucket_name): + return self.__client.clear_bucket(bucket_name) diff --git a/pyinfra/test/storage/client_mock.py b/pyinfra/test/storage/client_mock.py index 2b2fb26..c0966a4 100644 --- a/pyinfra/test/storage/client_mock.py +++ b/pyinfra/test/storage/client_mock.py @@ -17,5 +17,5 @@ class StorageClientMock: def get_all(self, bucket_name): return self.__data[bucket_name].values() - def purge(self, bucket_name): + def clear_bucket(self, bucket_name): self.__data[bucket_name] = {} diff --git a/pyinfra/test/unit_tests/storage_test.py b/pyinfra/test/unit_tests/storage_test.py index 712ae7b..957e11a 100644 --- a/pyinfra/test/unit_tests/storage_test.py +++ b/pyinfra/test/unit_tests/storage_test.py @@ -2,6 +2,7 @@ import logging import pytest +from pyinfra.config import CONFIG from pyinfra.exceptions import UnknownClient from pyinfra.storage.adapters.azure import AzureStorageAdapter from pyinfra.storage.adapters.s3 import S3StorageAdapter @@ -14,8 +15,8 @@ from pyinfra.test.storage.client_mock import StorageClientMock @pytest.mark.parametrize("client_name", ["mock", "azure", "s3"]) class TestStorage: - def test_purging_bucket_yields_empty_bucket(self, storage, bucket_name): - storage.purge(bucket_name) + def test_clearing_bucket_yields_empty_bucket(self, storage, bucket_name): + storage.clear_bucket(bucket_name) data_received = storage.get_all(bucket_name) assert not {*data_received} @@ -34,14 +35,13 @@ class TestStorage: storage.make_bucket(bucket_name) assert storage.bucket_exists(bucket_name) - def get_adapter(client_name): if client_name == "mock": return StorageAdapterMock(StorageClientMock()) if client_name == "azure": - return AzureStorageAdapter(get_azure_client()) + return AzureStorageAdapter(get_azure_client(CONFIG.test.azure)) if client_name == "s3": - return S3StorageAdapter(get_s3_client()) + return S3StorageAdapter(get_s3_client(CONFIG.test.minio)) else: raise UnknownClient(client_name) @@ -56,6 +56,6 @@ def adapter(client_name): def storage(client_name, bucket_name): storage = Storage(get_adapter(client_name)) storage.make_bucket(bucket_name) - storage.purge(bucket_name) + storage.clear_bucket(bucket_name) logging.info(client_name) return storage