793a427c50
Merge in RR/pyinfra from RED-6273-multi-tenant-storage to master
Squashed commit of the following:
commit 0fead1f8b59c9187330879b4e48d48355885c27c
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Tue Mar 28 15:02:22 2023 +0200
fix typos
commit 892a803726946876f8b8cd7905a0e73c419b2fb1
Author: Matthias Bisping <matthias.bisping@axbit.com>
Date: Tue Mar 28 14:41:49 2023 +0200
Refactoring
Replace custom storage caching logic with LRU decorator
commit eafcd90260731e3360ce960571f07dee8f521327
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 12:50:13 2023 +0100
fix bug in storage connection from endpoint
commit d0c9fb5b7d1c55ae2f90e8faa1efec9f7587c26a
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 11:49:34 2023 +0100
add logs to PayloadProcessor
- set log messages to determine if x-tenant
storage connection is working
commit 97309fe58037b90469cf7a3de342d4749a0edfde
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Fri Mar 24 10:41:59 2023 +0100
update PayloadProcessor
- introduce storage cache to make every unique
storage connection only once
- add functionality to pass optional processing
kwargs in queue message like the operation key to
the processing function
commit d48e8108fdc0d463c89aaa0d672061ab7dca83a0
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Wed Mar 22 13:34:43 2023 +0100
add multi-tenant storage connection 1st iteration
- forward x-tenant-id from queue message header to
payload processor
- add functions to receive storage infos from an
endpoint or the config. This enables hashing and
caching of connections created from these infos
- add function to initialize storage connections
from storage infos
- streamline and refactor tests to make them more
readable and robust and to make it easier to add
new tests
- update payload processor with first iteration
of multi tenancy storage connection support
with connection caching and backwards compability
commit 52c047c47b98e62d0b834a9b9b6c0e2bb0db41e5
Author: Julius Unverfehrt <julius.unverfehrt@iqser.com>
Date: Tue Mar 21 15:35:57 2023 +0100
add AES/GCM cipher functions
- decrypt x-tenant storage connection strings
84 lines
2.6 KiB
Python
84 lines
2.6 KiB
Python
import logging
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import testcontainers.compose
|
|
|
|
from pyinfra.config import get_config
|
|
from pyinfra.storage import get_storage_from_config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
TESTS_DIR = Path(__file__).resolve().parents[0]
|
|
|
|
|
|
@pytest.fixture(scope="session", autouse=True)
|
|
def docker_compose(sleep_seconds=30):
|
|
"""Note: `autouse` can be set to `False` while working on the code to speed up the testing. In that case, run
|
|
`docker-compose up` in the tests directory manually before running the tests.
|
|
"""
|
|
logger.info(f"Starting docker containers with {TESTS_DIR}/docker-compose.yml...")
|
|
compose = testcontainers.compose.DockerCompose(TESTS_DIR, compose_file_name="docker-compose.yml")
|
|
compose.start()
|
|
logger.info(f"Sleeping for {sleep_seconds} seconds to wait for containers to finish startup... ")
|
|
time.sleep(sleep_seconds)
|
|
yield compose
|
|
compose.stop()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def test_storage_config(storage_backend, bucket_name, monitoring_enabled):
|
|
config = get_config()
|
|
config.storage_backend = storage_backend
|
|
config.storage_bucket = bucket_name
|
|
config.storage_azureconnectionstring = "DefaultEndpointsProtocol=https;AccountName=iqserdevelopment;AccountKey=4imAbV9PYXaztSOMpIyAClg88bAZCXuXMGJG0GA1eIBpdh2PlnFGoRBnKqLy2YZUSTmZ3wJfC7tzfHtuC6FEhQ==;EndpointSuffix=core.windows.net"
|
|
config.monitoring_enabled = monitoring_enabled
|
|
config.prometheus_metric_prefix = "test"
|
|
config.prometheus_port = 8080
|
|
config.prometheus_host = "0.0.0.0"
|
|
return config
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def test_queue_config():
|
|
config = get_config()
|
|
config.rabbitmq_connection_sleep = 2
|
|
config.rabbitmq_heartbeat = 4
|
|
return config
|
|
|
|
|
|
@pytest.fixture
|
|
def payload(x_tenant_id):
|
|
x_tenant_entry = {"X-TENANT-ID": x_tenant_id} if x_tenant_id else {}
|
|
return {
|
|
"dossierId": "test",
|
|
"fileId": "test",
|
|
"targetFileExtension": "json.gz",
|
|
"responseFileExtension": "json.gz",
|
|
**x_tenant_entry,
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def response_payload():
|
|
return {
|
|
"dossierId": "test",
|
|
"fileId": "test",
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def storage(test_storage_config):
|
|
logger.debug("Setup for storage")
|
|
storage = get_storage_from_config(test_storage_config)
|
|
storage.make_bucket(test_storage_config.storage_bucket)
|
|
storage.clear_bucket(test_storage_config.storage_bucket)
|
|
yield storage
|
|
logger.debug("Teardown for storage")
|
|
try:
|
|
storage.clear_bucket(test_storage_config.storage_bucket)
|
|
except:
|
|
pass
|