From 3d9d2599ac7291053819e1f2e0fbff48c4146b3a Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Wed, 23 Mar 2022 15:56:29 +0100 Subject: [PATCH] banner refac; tests in container again; removed docker utils calls because does not work when running in container --- Dockerfile_tests | 2 +- .../src/main/resources/scripts/sonar-scan.sh | 12 ++-- banner.txt | 8 +++ pyinfra/config.py | 13 ----- pyinfra/locations.py | 2 + pyinfra/test/unit_tests/conftest.py | 20 +++---- pyinfra/utils/banner.py | 21 +++++++ pyinfra/utils/docker.py | 56 +++++++++---------- run_tests.sh | 13 +---- src/serve.py | 9 +-- 10 files changed, 82 insertions(+), 74 deletions(-) create mode 100644 banner.txt create mode 100644 pyinfra/utils/banner.py diff --git a/Dockerfile_tests b/Dockerfile_tests index b16eccc..4c7157f 100755 --- a/Dockerfile_tests +++ b/Dockerfile_tests @@ -16,4 +16,4 @@ COPY . . RUN python3 -m pip install -e . RUN python3 -m pip install -r requirements.txt -CMD coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml +CMD coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh index 8b109e2..e19025d 100755 --- a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -13,15 +13,13 @@ echo "coverage report generation" python3 -m pip install coverage # Install module & dependencies -python3 -m pip install -e . -python3 -m pip install -r requirements.txt +#python3 -m pip install -e . +#python3 -m pip install -r requirements.txt -#docker-compose -remove-orphans up -d -# -#echo "Sleeping for 30 seconds to wait for dependency containers to finish startup..." -#sleep 30s +bash run_tests.sh -coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml + +#coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml SERVICE_NAME=$1 diff --git a/banner.txt b/banner.txt new file mode 100644 index 0000000..e7851e0 --- /dev/null +++ b/banner.txt @@ -0,0 +1,8 @@ +______ _____ __ +| ___ \ |_ _| / _| +| |_/ / _ | | _ __ | |_ _ __ __ _ +| __/ | | || || '_ \| _| '__/ _` | +| | | |_| || || | | | | | | | (_| | +\_| \__, \___/_| |_|_| |_| \__,_| + __/ | + |___/ \ No newline at end of file diff --git a/pyinfra/config.py b/pyinfra/config.py index 5927171..1e94f08 100644 --- a/pyinfra/config.py +++ b/pyinfra/config.py @@ -5,19 +5,6 @@ from envyaml import EnvYAML from pyinfra.locations import CONFIG_FILE -def make_art(): - return """ -______ _____ __ -| ___ \ |_ _| / _| -| |_/ / _ | | _ __ | |_ _ __ __ _ -| __/ | | || || '_ \| _| '__/ _` | -| | | |_| || || | | | | | | | (_| | -\_| \__, \___/_| |_|_| |_| \__,_| - __/ | - |___/ -""" - - def _get_item_and_maybe_make_dotindexable(container, item): ret = container[item] return DotIndexable(ret) if isinstance(ret, dict) else ret diff --git a/pyinfra/locations.py b/pyinfra/locations.py index a1878da..abd6a88 100644 --- a/pyinfra/locations.py +++ b/pyinfra/locations.py @@ -14,3 +14,5 @@ CONFIG_FILE = PACKAGE_ROOT_DIR / "config.yaml" TEST_CONFIG_FILE = TEST_DIR / "config.yaml" COMPOSE_PATH = PACKAGE_ROOT_DIR + +BANNER_FILE = PACKAGE_ROOT_DIR / "banner.txt" diff --git a/pyinfra/test/unit_tests/conftest.py b/pyinfra/test/unit_tests/conftest.py index b6510a9..ce920b6 100644 --- a/pyinfra/test/unit_tests/conftest.py +++ b/pyinfra/test/unit_tests/conftest.py @@ -20,7 +20,7 @@ from pyinfra.test.config import CONFIG from pyinfra.test.queue.queue_manager_mock import QueueManagerMock from pyinfra.test.storage.adapter_mock import StorageAdapterMock from pyinfra.test.storage.client_mock import StorageClientMock -from pyinfra.utils import docker +# from pyinfra.utils import docker from pyinfra.visitor import StorageStrategy, ForwardingStrategy, QueueVisitor logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ def storage(client_name, bucket_name, request): @pytest.fixture(scope="session", autouse=True) def docker_compose(sleep_seconds=30): logger.info(f"Starting docker containers with {COMPOSE_PATH}/docker-compose.yml...") - compose = testcontainers.compose.DockerCompose(COMPOSE_PATH) + compose = testcontainers.compose.DockerCompose(COMPOSE_PATH, compose_file_name="docker-compose.yml") compose.start() logger.info(f"Sleeping for {sleep_seconds} seconds to wait for containers to finish startup... ") time.sleep(sleep_seconds) @@ -81,24 +81,24 @@ def docker_compose(sleep_seconds=30): compose.stop() -def get_endpoint_from_running_container(container_name, config): - port = config.endpoint.split(":")[-1] - endpoint = f"{docker.get_ip_address(container_name)}:{port}" - return endpoint +# def get_endpoint_from_running_container(container_name, config): +# port = config.endpoint.split(":")[-1] +# endpoint = f"{docker.get_ip_address(container_name)}:{port}" +# return endpoint def get_pika_connection_params(): params = get_connection_params() - params.host = docker.get_ip_address("rabbitmq") + # params.host = docker.get_ip_address("rabbitmq") return params def get_s3_params(s3_backend): params = CONFIG.storage[s3_backend] - if s3_backend == "minio": - port = params.endpoint.split(":")[-1] - params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}" + # if s3_backend == "minio": + # port = params.endpoint.split(":")[-1] + # params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}" return params diff --git a/pyinfra/utils/banner.py b/pyinfra/utils/banner.py new file mode 100644 index 0000000..20d066e --- /dev/null +++ b/pyinfra/utils/banner.py @@ -0,0 +1,21 @@ +import logging + +from pyinfra.locations import BANNER_FILE + + +def show_banner(): + with open(BANNER_FILE) as f: + banner = "\n" + "".join(f.readlines()) + "\n" + + logger = logging.getLogger(__name__) + logger.propagate = False + + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + + formatter = logging.Formatter("") + + handler.setFormatter(formatter) + logger.addHandler(handler) + + logger.info(banner) diff --git a/pyinfra/utils/docker.py b/pyinfra/utils/docker.py index ce3443b..6370e33 100644 --- a/pyinfra/utils/docker.py +++ b/pyinfra/utils/docker.py @@ -1,28 +1,28 @@ -import docker - -from pyinfra.exceptions import NoSuchContainer - - -def filter_by_name(containers, container_name, exact=False): - def matches(container): - return container_name == container.name if exact else container_name in container.name - - containers = filter(matches, containers) - return containers - - -def get_container(container_name, exact=False): - client = docker.from_env() - containers = client.containers.list() - containers = filter_by_name(containers, container_name, exact=exact) - - try: - return next(containers) - except StopIteration: - raise NoSuchContainer(f"No container found for '{container_name}'") - - -def get_ip_address(container_name, exact=False): - container = get_container(container_name, exact=exact) - ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] - return ip_address +# import docker +# +# from pyinfra.exceptions import NoSuchContainer +# +# +# def filter_by_name(containers, container_name, exact=False): +# def matches(container): +# return container_name == container.name if exact else container_name in container.name +# +# containers = filter(matches, containers) +# return containers +# +# +# def get_container(container_name, exact=False): +# client = docker.from_env() +# containers = client.containers.list() +# containers = filter_by_name(containers, container_name, exact=exact) +# +# try: +# return next(containers) +# except StopIteration: +# raise NoSuchContainer(f"No container found for '{container_name}'") +# +# +# def get_ip_address(container_name, exact=False): +# container = get_container(container_name, exact=exact) +# ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"] +# return ip_address diff --git a/run_tests.sh b/run_tests.sh index fc81e96..9bea183 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,16 +1,7 @@ echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 -docker build -f Dockerfile_tests -t pyinfra-tests:dev . +docker build -f Dockerfile_tests -t pyinfra-tests:dev . --build-arg BASE_ROOT="" docker tag pyinfra-tests:dev nexus.iqser.com:5001/red/pyinfra-tests:dev -docker push nexus.iqser.com:5001/red/pyinfra-tests:dev - -source build_venv/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install docker-compose - -docker-compose -remove-orphans up -d - -echo "Sleeping for 30 seconds to wait for dependency containers to finish startup..." -sleep 30s +#docker push nexus.iqser.com:5001/red/pyinfra-tests:dev rnd=$(date +"%s") name=pyinfra-tests-${rnd} diff --git a/src/serve.py b/src/serve.py index 8b76724..850933f 100644 --- a/src/serve.py +++ b/src/serve.py @@ -4,12 +4,13 @@ from multiprocessing import Process import requests from retry import retry -from pyinfra.config import CONFIG, make_art +from pyinfra.config import CONFIG from pyinfra.exceptions import AnalysisFailure, ConsumerError from pyinfra.flask import run_probing_webserver, set_up_probing_webserver from pyinfra.queue.consumer import Consumer from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager from pyinfra.storage.storages import get_storage +from pyinfra.utils.banner import show_banner from pyinfra.visitor import QueueVisitor, StorageStrategy @@ -37,9 +38,9 @@ def make_callback(analysis_endpoint): def main(): + show_banner() webserver = Process(target=run_probing_webserver, args=(set_up_probing_webserver(),)) - logging.info(make_art()) logging.info("Starting webserver...") webserver.start() @@ -52,11 +53,11 @@ def main(): @retry(ConsumerError, tries=3, delay=5, jitter=(1, 3)) def consume(): + consumer = Consumer(visitor, queue_manager) try: - consumer = Consumer(visitor, queue_manager) consumer.consume_and_publish() except Exception as err: - raise ConsumerError from err + raise ConsumerError() from err try: consume()