This commit is contained in:
Matthias Bisping 2022-02-25 11:44:46 +01:00
parent 23b35b1f91
commit 81be5e55b6
8 changed files with 14 additions and 11 deletions

View File

@ -30,5 +30,5 @@ class StorageAdapter(ABC):
pass
@abstractmethod
def list_bucket_files(self, bucket_name):
def get_all_object_names(self, bucket_name):
pass

View File

@ -73,7 +73,7 @@ class AzureStorageAdapter(StorageAdapter):
blobs = container_client.list_blobs()
container_client.delete_blobs(*blobs)
def list_bucket_files(self, bucket_name):
def get_all_object_names(self, bucket_name):
container_client = self.__provide_container_client(bucket_name)
blobs = container_client.list_blobs()
return zip(repeat(bucket_name), map(attrgetter("name"), blobs))

View File

@ -50,6 +50,6 @@ class S3StorageAdapter(StorageAdapter):
for obj in objects:
self.__client.remove_object(bucket_name, obj.object_name)
def list_bucket_files(self, bucket_name):
def get_all_object_names(self, bucket_name):
objs = self.__client.list_objects(bucket_name, recursive=True)
return zip(repeat(bucket_name), map(attrgetter("object_name"), objs))

View File

@ -1,5 +1,8 @@
from pyinfra.storage.adapters.adapter import StorageAdapter
class Storage:
def __init__(self, adapter):
def __init__(self, adapter: StorageAdapter):
self.__adapter = adapter
def make_bucket(self, bucket_name):
@ -20,5 +23,5 @@ class Storage:
def clear_bucket(self, bucket_name):
return self.__adapter.clear_bucket(bucket_name)
def list_bucket_files(self, bucket_name):
return self.__adapter.list_bucket_files(bucket_name)
def get_all_object_names(self, bucket_name):
return self.__adapter.get_all_object_names(bucket_name)

View File

@ -26,5 +26,5 @@ class StorageAdapterMock(StorageAdapter):
def clear_bucket(self, bucket_name):
return self.__client.clear_bucket(bucket_name)
def list_bucket_files(self, bucket_name):
return self.__client.list_bucket_files(bucket_name)
def get_all_object_names(self, bucket_name):
return self.__client.get_all_object_names(bucket_name)

View File

@ -23,5 +23,5 @@ class StorageClientMock:
def clear_bucket(self, bucket_name):
self.__data[bucket_name] = {}
def list_bucket_files(self, bucket_name):
def get_all_object_names(self, bucket_name):
return zip(repeat(bucket_name), self.__data[bucket_name])

View File

@ -48,7 +48,7 @@ class TestStorage:
def test_listing_bucket_files_yields_all_files_in_bucket(self, storage, bucket_name):
storage.put(bucket_name, "file1", b"content 1")
storage.put(bucket_name, "file2", b"content 2")
full_names_received = storage.list_bucket_files(bucket_name)
full_names_received = storage.get_all_object_names(bucket_name)
assert {(bucket_name, "file1"), (bucket_name, "file2")} == {*full_names_received}

View File

@ -7,7 +7,7 @@ from pyinfra.storage.storages import get_s3_storage
def build_message_bodies():
storage = get_s3_storage()
for bucket_name, pdf_name in storage.list_bucket_files(CONFIG.test.bucket):
for bucket_name, pdf_name in storage.get_all_object_names(CONFIG.test.bucket):
file_id = pdf_name.split(".")[0]
dossier_id, file_id = file_id.split("/")
yield json.dumps({"dossierId": dossier_id, "fileId": file_id})