added s3 config changes from image-service

config.yaml

@@ -16,11 +16,19 @@ rabbitmq:
     enabled: $RETRY|False  # Toggles retry behaviour
     max_attempts: $MAX_ATTEMPTS|3  # Number of times a message may fail before being published to dead letter queue
 
-minio:
-  endpoint: $STORAGE_ENDPOINT|"127.0.0.1:9000"  # MinIO endpoint
-  user: $STORAGE_KEY|root  # MinIO user name
-  password: $STORAGE_SECRET|password  # MinIO user password
-  bucket: $STORAGE_BUCKET_NAME|redaction  # MinIO bucket
+s3:
+  user: $STORAGE_KEY|root  # s3 user name
+  password: $STORAGE_SECRET|password  # s3 user password
+  bucket: $STORAGE_BUCKET_NAME|redaction  # s3 bucket
+
+  aws:
+    endpoint: $STORAGE_ENDPOINT|null  # AWS endpoint
+
+  minio:
+    host: $STORAGE_HOST|localhost  # MinIO host address
+    port: $STORAGE_PORT|9000  # MinIO host port
+
+  backend: $S3_BACKEND|minio  # S3 backend to use
 
 azure_blob_storage:
   connection_string: $STORAGE_AZURECONNECTIONSTRING|"DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"

pyinfra/config.py

@@ -24,6 +24,9 @@ class DotIndexable:
     def __repr__(self):
         return self.x.__repr__()
 
+    def __getitem__(self, item):
+        return self.__getattr__(item)
+
 
 class Config:
     def __init__(self, config_path):

pyinfra/storage/minio.py

@@ -8,31 +8,48 @@ from pyinfra.config import CONFIG
 from pyinfra.storage.storage import StorageHandle
 
 
-def get_minio_client(access_key=None, secret_key=None) -> Minio:
+def get_minio_client(access_key=None, secret_key=None, backend=None) -> Minio:
     """Instantiates a minio.Minio client.
 
     Args:
         access_key: Access key for MinIO client (username).
         secret_key: Secret key for MinIO client (password).
+        backend: Whether to use MinIO or AWS
 
     Returns:
         A minio.Minio client.
     """
-    access_key = CONFIG.minio.user if access_key is None else access_key
-    secret_key = CONFIG.minio.password if secret_key is None else secret_key
+    access_key = CONFIG.s3.user if access_key is None else access_key
+    secret_key = CONFIG.s3.password if secret_key is None else secret_key
+    backend = CONFIG.s3.backend if backend is None else backend
 
-    # TODO: secure=True/False?
-    return Minio(CONFIG.minio.endpoint, access_key=access_key, secret_key=secret_key, secure=False)
+    assert backend in ["aws", "minio"]
+
+    if backend == "aws":
+        prefix, endpoint = CONFIG.s3.aws.endpoint.split("://")
+        assert prefix in ["http", "https"]
+        secure = prefix == "https"
+
+    else:
+        endpoint = f"{CONFIG.s3.minio.host}:{CONFIG.s3.minio.port}"
+        secure = False
+
+    return Minio(
+        endpoint=endpoint,
+        access_key=access_key,
+        secret_key=secret_key,
+        secure=secure
+    )
 
 
 class MinioHandle(StorageHandle):
     """Wrapper around a MinIO client that provides operations on the MinIO store required by the service."""
 
-    def __init__(self):
+    def __init__(self, backend = None):
         """Initializes a MinioHandle"""
         super().__init__()
-        self.client: Minio = get_minio_client()
-        self.default_container_name = CONFIG.minio.bucket
+        self.client: Minio = get_minio_client(backend=backend)
+        self.default_container_name = CONFIG.s3.bucket
         self.backend = "s3"
 
     def _StorageHandle__provide_container(self, container_name):

src/serve.py

@@ -23,6 +23,7 @@ from pyinfra.storage.minio import MinioHandle
 def get_storage():
 
     storage_backend = CONFIG.service.storage_backend
+
     if storage_backend == "s3":
         storage = MinioHandle()
     elif storage_backend == "azure":