diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh index 10d25ab..f04280f 100755 --- a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -9,7 +9,14 @@ python3 -m pip install --upgrade pip python3 -m pip install dependency-check echo "coverage report generation" -bash run_tests.sh + +python3 -m pip install coverage + +# Install module & dependencies +python3 -m pip install -e . +python3 -m pip install -r requirements.txt + +coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml SERVICE_NAME=$1 diff --git a/pyinfra/exceptions.py b/pyinfra/exceptions.py index 9b37df5..474cc6f 100644 --- a/pyinfra/exceptions.py +++ b/pyinfra/exceptions.py @@ -24,3 +24,7 @@ class UnknownClient(ValueError): class ConsumerError(Exception): pass + + +class NoSuchContainer(KeyError): + pass diff --git a/pyinfra/queue/queue_manager/pika_queue_manager.py b/pyinfra/queue/queue_manager/pika_queue_manager.py index 6cfae52..ca7f9c6 100644 --- a/pyinfra/queue/queue_manager/pika_queue_manager.py +++ b/pyinfra/queue/queue_manager/pika_queue_manager.py @@ -41,17 +41,13 @@ def monkey_patch_queue_handle(channel, queue) -> QueueHandle: return queue_handle -def get_connection(): +def get_connection_params(): credentials = pika.PlainCredentials(username=CONFIG.rabbitmq.user, password=CONFIG.rabbitmq.password) - kwargs = {"host": CONFIG.rabbitmq.host, "port": CONFIG.rabbitmq.port, "credentials": credentials} - parameters = pika.ConnectionParameters(**kwargs) - connection = pika.BlockingConnection(parameters=parameters) - - return connection + return parameters def get_n_previous_attempts(props): @@ -63,10 +59,13 @@ def attempts_remain(n_attempts, max_attempts): class PikaQueueManager(QueueManager): - def __init__(self, input_queue, output_queue, dead_letter_queue=None): + def __init__(self, input_queue, output_queue, dead_letter_queue=None, connection_params=None): super().__init__(input_queue, output_queue) - self.connection = get_connection() + if not connection_params: + connection_params = get_connection_params() + + self.connection = pika.BlockingConnection(parameters=connection_params) self.channel = self.connection.channel() if not dead_letter_queue: diff --git a/pyinfra/storage/clients/s3.py b/pyinfra/storage/clients/s3.py index 19ee87e..943d88c 100644 --- a/pyinfra/storage/clients/s3.py +++ b/pyinfra/storage/clients/s3.py @@ -1,4 +1,3 @@ -import logging import re from minio import Minio @@ -9,6 +8,7 @@ from pyinfra.exceptions import InvalidEndpoint def parse_endpoint(endpoint): # FIXME Greedy matching (.+) since we get random storage names on kubernetes (eg http://red-research-headless:9000) + # FIXME this has been broken and accepts invalid URLs endpoint_pattern = r"(?Phttps?)*(?:://)*(?P
(?:(?:(?:\d{1,3}\.){3}\d{1,3})|.+)(?:\:\d+)?)" match = re.match(endpoint_pattern, endpoint) diff --git a/pyinfra/test/unit_tests/conftest.py b/pyinfra/test/unit_tests/conftest.py index 46b3ac5..9a7b67d 100644 --- a/pyinfra/test/unit_tests/conftest.py +++ b/pyinfra/test/unit_tests/conftest.py @@ -6,7 +6,7 @@ import pytest from pyinfra.exceptions import UnknownClient from pyinfra.locations import TEST_DIR -from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager +from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager, get_connection_params from pyinfra.queue.queue_manager.queue_manager import QueueManager from pyinfra.storage.adapters.azure import AzureStorageAdapter from pyinfra.storage.adapters.s3 import S3StorageAdapter @@ -17,6 +17,7 @@ from pyinfra.test.config import CONFIG from pyinfra.test.queue.queue_manager_mock import QueueManagerMock from pyinfra.test.storage.adapter_mock import StorageAdapterMock from pyinfra.test.storage.client_mock import StorageClientMock +from pyinfra.utils import docker from pyinfra.visitor import StorageStrategy, ForwardingStrategy, QueueVisitor logger = logging.getLogger(__name__) @@ -66,13 +67,35 @@ def storage(client_name, bucket_name, request): storage.clear_bucket(bucket_name) +def get_endpoint_from_running_container(container_name, config): + port = config.endpoint.split(":")[-1] + endpoint = f"{docker.get_ip_address(container_name)}:{port}" + return endpoint + + +def get_pika_connection_params(): + params = get_connection_params() + params.host = docker.get_ip_address("rabbitmq") + return params + + +def get_s3_params(s3_backend): + params = CONFIG.storage[s3_backend] + + if s3_backend == "minio": + port = params.endpoint.split(":")[-1] + params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}" + + return params + + def get_adapter(client_name, s3_backend): if client_name == "mock": return StorageAdapterMock(StorageClientMock()) if client_name == "azure": return AzureStorageAdapter(get_azure_client(CONFIG.storage.azure.connection_string)) if client_name == "s3": - return S3StorageAdapter(get_s3_client(CONFIG.storage[s3_backend])) + return S3StorageAdapter(get_s3_client(get_s3_params(s3_backend))) else: raise UnknownClient(client_name) @@ -81,7 +104,7 @@ def get_queue_manager(queue_manager_name) -> QueueManager: if queue_manager_name == "mock": return QueueManagerMock("input", "output") if queue_manager_name == "pika": - return PikaQueueManager("input", "output") + return PikaQueueManager("input", "output", connection_params=get_pika_connection_params()) @pytest.fixture(scope="session") diff --git a/pyinfra/utils/__init__.py b/pyinfra/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyinfra/utils/docker.py b/pyinfra/utils/docker.py new file mode 100644 index 0000000..ce3443b --- /dev/null +++ b/pyinfra/utils/docker.py @@ -0,0 +1,28 @@ +import docker + +from pyinfra.exceptions import NoSuchContainer + + +def filter_by_name(containers, container_name, exact=False): + def matches(container): + return container_name == container.name if exact else container_name in container.name + + containers = filter(matches, containers) + return containers + + +def get_container(container_name, exact=False): + client = docker.from_env() + containers = client.containers.list() + containers = filter_by_name(containers, container_name, exact=exact) + + try: + return next(containers) + except StopIteration: + raise NoSuchContainer(f"No container found for '{container_name}'") + + +def get_ip_address(container_name, exact=False): + container = get_container(container_name, exact=exact) + ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] + return ip_address diff --git a/requirements.txt b/requirements.txt index c13ae24..c4079a6 100755 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ requests==2.27.1 # dev docker-compose==1.29.2 tqdm==4.62.3 -dependency-check pyinfra~=0.0.1 pytest~=7.0.1 \ No newline at end of file