diff --git a/config.yaml b/config.yaml index 5cbe777..ac27215 100755 --- a/config.yaml +++ b/config.yaml @@ -16,11 +16,19 @@ rabbitmq: enabled: $RETRY|False # Toggles retry behaviour max_attempts: $MAX_ATTEMPTS|3 # Number of times a message may fail before being published to dead letter queue -minio: - endpoint: $STORAGE_ENDPOINT|"127.0.0.1:9000" # MinIO endpoint - user: $STORAGE_KEY|root # MinIO user name - password: $STORAGE_SECRET|password # MinIO user password - bucket: $STORAGE_BUCKET_NAME|redaction # MinIO bucket +s3: + user: $STORAGE_KEY|root # s3 user name + password: $STORAGE_SECRET|password # s3 user password + bucket: $STORAGE_BUCKET_NAME|redaction # s3 bucket + + aws: + endpoint: $STORAGE_ENDPOINT|null # AWS endpoint + + minio: + host: $STORAGE_HOST|localhost # MinIO host address + port: $STORAGE_PORT|9000 # MinIO host port + + backend: $S3_BACKEND|minio # S3 backend to use azure_blob_storage: connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net" diff --git a/pyinfra/config.py b/pyinfra/config.py index 2e5f071..c613a49 100644 --- a/pyinfra/config.py +++ b/pyinfra/config.py @@ -24,6 +24,9 @@ class DotIndexable: def __repr__(self): return self.x.__repr__() + def __getitem__(self, item): + return self.__getattr__(item) + class Config: def __init__(self, config_path): diff --git a/pyinfra/storage/minio.py b/pyinfra/storage/minio.py index 5f4f4b4..d331617 100644 --- a/pyinfra/storage/minio.py +++ b/pyinfra/storage/minio.py @@ -8,31 +8,48 @@ from pyinfra.config import CONFIG from pyinfra.storage.storage import StorageHandle -def get_minio_client(access_key=None, secret_key=None) -> Minio: +def get_minio_client(access_key=None, secret_key=None, backend=None) -> Minio: """Instantiates a minio.Minio client. Args: access_key: Access key for MinIO client (username). secret_key: Secret key for MinIO client (password). + backend: Whether to use MinIO or AWS Returns: A minio.Minio client. """ - access_key = CONFIG.minio.user if access_key is None else access_key - secret_key = CONFIG.minio.password if secret_key is None else secret_key + access_key = CONFIG.s3.user if access_key is None else access_key + secret_key = CONFIG.s3.password if secret_key is None else secret_key + backend = CONFIG.s3.backend if backend is None else backend - # TODO: secure=True/False? - return Minio(CONFIG.minio.endpoint, access_key=access_key, secret_key=secret_key, secure=False) + assert backend in ["aws", "minio"] + + if backend == "aws": + prefix, endpoint = CONFIG.s3.aws.endpoint.split("://") + assert prefix in ["http", "https"] + secure = prefix == "https" + + else: + endpoint = f"{CONFIG.s3.minio.host}:{CONFIG.s3.minio.port}" + secure = False + + return Minio( + endpoint=endpoint, + access_key=access_key, + secret_key=secret_key, + secure=secure + ) class MinioHandle(StorageHandle): """Wrapper around a MinIO client that provides operations on the MinIO store required by the service.""" - def __init__(self): + def __init__(self, backend = None): """Initializes a MinioHandle""" super().__init__() - self.client: Minio = get_minio_client() - self.default_container_name = CONFIG.minio.bucket + self.client: Minio = get_minio_client(backend=backend) + self.default_container_name = CONFIG.s3.bucket self.backend = "s3" def _StorageHandle__provide_container(self, container_name): diff --git a/src/serve.py b/src/serve.py index a37f3b4..7376b93 100644 --- a/src/serve.py +++ b/src/serve.py @@ -23,6 +23,7 @@ from pyinfra.storage.minio import MinioHandle def get_storage(): storage_backend = CONFIG.service.storage_backend + if storage_backend == "s3": storage = MinioHandle() elif storage_backend == "azure":