793a427c50
Merge in RR/pyinfra from RED-6273-multi-tenant-storage to master
Squashed commit of the following:
commit 0fead1f8b59c9187330879b4e48d48355885c27c
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Tue Mar 28 15:02:22 2023 +0200
fix typos
commit 892a803726946876f8b8cd7905a0e73c419b2fb1
Author: Matthias Bisping <matthias.bisping@axbit.com>
Date: Tue Mar 28 14:41:49 2023 +0200
Refactoring
Replace custom storage caching logic with LRU decorator
commit eafcd90260731e3360ce960571f07dee8f521327
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 12:50:13 2023 +0100
fix bug in storage connection from endpoint
commit d0c9fb5b7d1c55ae2f90e8faa1efec9f7587c26a
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 11:49:34 2023 +0100
add logs to PayloadProcessor
- set log messages to determine if x-tenant
storage connection is working
commit 97309fe58037b90469cf7a3de342d4749a0edfde
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 10:41:59 2023 +0100
update PayloadProcessor
- introduce storage cache to make every unique
storage connection only once
- add functionality to pass optional processing
kwargs in queue message like the operation key to
the processing function
commit d48e8108fdc0d463c89aaa0d672061ab7dca83a0
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Wed Mar 22 13:34:43 2023 +0100
add multi-tenant storage connection 1st iteration
- forward x-tenant-id from queue message header to
payload processor
- add functions to receive storage infos from an
endpoint or the config. This enables hashing and
caching of connections created from these infos
- add function to initialize storage connections
from storage infos
- streamline and refactor tests to make them more
readable and robust and to make it easier to add
new tests
- update payload processor with first iteration
of multi tenancy storage connection support
with connection caching and backwards compability
commit 52c047c47b98e62d0b834a9b9b6c0e2bb0db41e5
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Tue Mar 21 15:35:57 2023 +0100
add AES/GCM cipher functions
- decrypt x-tenant storage connection strings
49 lines
1.6 KiB
Python
49 lines
1.6 KiB
Python
from functools import lru_cache, partial
|
|
from typing import Callable
|
|
|
|
from funcy import compose
|
|
|
|
from pyinfra.config import Config
|
|
from pyinfra.storage.storage_info import get_storage_info_from_config, get_storage_from_storage_info
|
|
from pyinfra.storage.storages.interface import Storage
|
|
from pyinfra.utils.compressing import get_decompressor, get_compressor
|
|
from pyinfra.utils.encoding import get_decoder, get_encoder
|
|
|
|
|
|
def get_storage_from_config(config: Config) -> Storage:
|
|
|
|
storage_info = get_storage_info_from_config(config)
|
|
storage = get_storage_from_storage_info(storage_info)
|
|
|
|
return storage
|
|
|
|
|
|
def verify_existence(storage: Storage, bucket: str, file_name: str) -> str:
|
|
if not storage.exists(bucket, file_name):
|
|
raise FileNotFoundError(f"{file_name=} name not found on storage in {bucket=}.")
|
|
return file_name
|
|
|
|
|
|
@lru_cache(maxsize=10)
|
|
def make_downloader(storage: Storage, bucket: str, file_type: str, compression_type: str) -> Callable:
|
|
|
|
verify = partial(verify_existence, storage, bucket)
|
|
download = partial(storage.get_object, bucket)
|
|
decompress = get_decompressor(compression_type)
|
|
decode = get_decoder(file_type)
|
|
|
|
return compose(decode, decompress, download, verify)
|
|
|
|
|
|
@lru_cache(maxsize=10)
|
|
def make_uploader(storage: Storage, bucket: str, file_type: str, compression_type: str) -> Callable:
|
|
|
|
upload = partial(storage.put_object, bucket)
|
|
compress = get_compressor(compression_type)
|
|
encode = get_encoder(file_type)
|
|
|
|
def inner(file_name, file_bytes):
|
|
upload(file_name, compose(compress, encode)(file_bytes))
|
|
|
|
return inner
|