tests not run in container again... switched to manual address management, because of dependencies to sibling containers (minio, rabbitmq), running on non-config endpoints and docker addresses cannot be queried from inside test container

bamboo-specs/src/main/resources/scripts/sonar-scan.sh

@@ -9,7 +9,14 @@ python3 -m pip install --upgrade pip
 python3 -m pip install dependency-check
 
 echo "coverage report generation"
-bash run_tests.sh
+
+python3 -m pip install coverage
+
+# Install module & dependencies
+python3 -m pip install -e .
+python3 -m pip install -r requirements.txt
+
+coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml
 
 SERVICE_NAME=$1
 

pyinfra/exceptions.py

@@ -24,3 +24,7 @@ class UnknownClient(ValueError):
 
 class ConsumerError(Exception):
     pass
+
+
+class NoSuchContainer(KeyError):
+    pass

pyinfra/queue/queue_manager/pika_queue_manager.py

@@ -41,17 +41,13 @@ def monkey_patch_queue_handle(channel, queue) -> QueueHandle:
     return queue_handle
 
 
-def get_connection():
+def get_connection_params():
 
     credentials = pika.PlainCredentials(username=CONFIG.rabbitmq.user, password=CONFIG.rabbitmq.password)
-
     kwargs = {"host": CONFIG.rabbitmq.host, "port": CONFIG.rabbitmq.port, "credentials": credentials}
-
     parameters = pika.ConnectionParameters(**kwargs)
 
-    connection = pika.BlockingConnection(parameters=parameters)
-
-    return connection
+    return parameters
 
 
 def get_n_previous_attempts(props):
@@ -63,10 +59,13 @@ def attempts_remain(n_attempts, max_attempts):
 
 
 class PikaQueueManager(QueueManager):
-    def __init__(self, input_queue, output_queue, dead_letter_queue=None):
+    def __init__(self, input_queue, output_queue, dead_letter_queue=None, connection_params=None):
         super().__init__(input_queue, output_queue)
 
-        self.connection = get_connection()
+        if not connection_params:
+            connection_params = get_connection_params()
+
+        self.connection = pika.BlockingConnection(parameters=connection_params)
         self.channel = self.connection.channel()
 
         if not dead_letter_queue:

pyinfra/storage/clients/s3.py

@@ -1,4 +1,3 @@
-import logging
 import re
 
 from minio import Minio
@@ -9,6 +8,7 @@ from pyinfra.exceptions import InvalidEndpoint
 
 def parse_endpoint(endpoint):
     # FIXME Greedy matching (.+) since we get random storage names on kubernetes (eg http://red-research-headless:9000)
+    # FIXME this has been broken and accepts invalid URLs
     endpoint_pattern = r"(?P<protocol>https?)*(?:://)*(?P<address>(?:(?:(?:\d{1,3}\.){3}\d{1,3})|.+)(?:\:\d+)?)"
 
     match = re.match(endpoint_pattern, endpoint)

pyinfra/test/unit_tests/conftest.py

@@ -6,7 +6,7 @@ import pytest
 
 from pyinfra.exceptions import UnknownClient
 from pyinfra.locations import TEST_DIR
-from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager
+from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager, get_connection_params
 from pyinfra.queue.queue_manager.queue_manager import QueueManager
 from pyinfra.storage.adapters.azure import AzureStorageAdapter
 from pyinfra.storage.adapters.s3 import S3StorageAdapter
@@ -17,6 +17,7 @@ from pyinfra.test.config import CONFIG
 from pyinfra.test.queue.queue_manager_mock import QueueManagerMock
 from pyinfra.test.storage.adapter_mock import StorageAdapterMock
 from pyinfra.test.storage.client_mock import StorageClientMock
+from pyinfra.utils import docker
 from pyinfra.visitor import StorageStrategy, ForwardingStrategy, QueueVisitor
 
 logger = logging.getLogger(__name__)
@@ -66,13 +67,35 @@ def storage(client_name, bucket_name, request):
     storage.clear_bucket(bucket_name)
 
 
+def get_endpoint_from_running_container(container_name, config):
+    port = config.endpoint.split(":")[-1]
+    endpoint = f"{docker.get_ip_address(container_name)}:{port}"
+    return endpoint
+
+
+def get_pika_connection_params():
+    params = get_connection_params()
+    params.host = docker.get_ip_address("rabbitmq")
+    return params
+
+
+def get_s3_params(s3_backend):
+    params = CONFIG.storage[s3_backend]
+
+    if s3_backend == "minio":
+        port = params.endpoint.split(":")[-1]
+        params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}"
+
+    return params
+
+
 def get_adapter(client_name, s3_backend):
     if client_name == "mock":
         return StorageAdapterMock(StorageClientMock())
     if client_name == "azure":
         return AzureStorageAdapter(get_azure_client(CONFIG.storage.azure.connection_string))
     if client_name == "s3":
-        return S3StorageAdapter(get_s3_client(CONFIG.storage[s3_backend]))
+        return S3StorageAdapter(get_s3_client(get_s3_params(s3_backend)))
     else:
         raise UnknownClient(client_name)
 
@@ -81,7 +104,7 @@ def get_queue_manager(queue_manager_name) -> QueueManager:
     if queue_manager_name == "mock":
         return QueueManagerMock("input", "output")
     if queue_manager_name == "pika":
-        return PikaQueueManager("input", "output")
+        return PikaQueueManager("input", "output", connection_params=get_pika_connection_params())
 
 
 @pytest.fixture(scope="session")

pyinfra/utils/docker.py

@@ -0,0 +1,28 @@
+import docker
+
+from pyinfra.exceptions import NoSuchContainer
+
+
+def filter_by_name(containers, container_name, exact=False):
+    def matches(container):
+        return container_name == container.name if exact else container_name in container.name
+
+    containers = filter(matches, containers)
+    return containers
+
+
+def get_container(container_name, exact=False):
+    client = docker.from_env()
+    containers = client.containers.list()
+    containers = filter_by_name(containers, container_name, exact=exact)
+
+    try:
+        return next(containers)
+    except StopIteration:
+        raise NoSuchContainer(f"No container found for '{container_name}'")
+
+
+def get_ip_address(container_name, exact=False):
+    container = get_container(container_name, exact=exact)
+    ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
+    return ip_address

requirements.txt

@@ -10,7 +10,6 @@ requests==2.27.1
 # dev
 docker-compose==1.29.2
 tqdm==4.62.3
-dependency-check
 
 pyinfra~=0.0.1
 pytest~=7.0.1