Marmelator/pyinfra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 36 Wiki Activity

banner refac; tests in container again; removed docker utils calls because does not work when running in container

Browse Source
This commit is contained in:
Matthias Bisping 2022-03-23 15:56:29 +01:00
parent e3c3482054
commit 3d9d2599ac
10 changed files with 82 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile_tests
View File

@ -16,4 +16,4 @@ COPY . .
RUN python3 -m pip install -e .
RUN python3 -m pip install -r requirements.txt
CMD coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml
CMD coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml

12
bamboo-specs/src/main/resources/scripts/sonar-scan.sh
View File

@ -13,15 +13,13 @@ echo "coverage report generation"
python3 -m pip install coverage
# Install module & dependencies
python3 -m pip install -e .
python3 -m pip install -r requirements.txt
#python3 -m pip install -e .
#python3 -m pip install -r requirements.txt
#docker-compose -remove-orphans up -d
#
#echo "Sleeping for 30 seconds to wait for dependency containers to finish startup..."
#sleep 30s
bash run_tests.sh
coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml
#coverage run -m pytest pyinfra/test/ -x && coverage report -m && coverage xml
SERVICE_NAME=$1

8
banner.txt Normal file
View File

@ -0,0 +1,8 @@
______ _____ __
| ___ \ |_ _| / _|
| |_/ / _ | | _ __ | |_ _ __ __ _
| __/ | | || || '_ \| _| '__/ _` |
| | | |_| || || | | | | | | | (_| |
\_| \__, \___/_| |_|_| |_| \__,_|
__/ |
|___/

13
pyinfra/config.py
View File

@ -5,19 +5,6 @@ from envyaml import EnvYAML
from pyinfra.locations import CONFIG_FILE
def make_art():
return """
______ _____ __
| ___ \ |_ _| / _|
| |_/ / _ | | _ __ | |_ _ __ __ _
| __/ | | || || '_ \| _| '__/ _` |
| | | |_| || || | | | | | | | (_| |
\_| \__, \___/_| |_|_| |_| \__,_|
__/ |
|___/
"""
def _get_item_and_maybe_make_dotindexable(container, item):
ret = container[item]
return DotIndexable(ret) if isinstance(ret, dict) else ret

2
pyinfra/locations.py
View File

@ -14,3 +14,5 @@ CONFIG_FILE = PACKAGE_ROOT_DIR / "config.yaml"
TEST_CONFIG_FILE = TEST_DIR / "config.yaml"
COMPOSE_PATH = PACKAGE_ROOT_DIR
BANNER_FILE = PACKAGE_ROOT_DIR / "banner.txt"

20
pyinfra/test/unit_tests/conftest.py
View File

@ -20,7 +20,7 @@ from pyinfra.test.config import CONFIG
from pyinfra.test.queue.queue_manager_mock import QueueManagerMock
from pyinfra.test.storage.adapter_mock import StorageAdapterMock
from pyinfra.test.storage.client_mock import StorageClientMock
from pyinfra.utils import docker
# from pyinfra.utils import docker
from pyinfra.visitor import StorageStrategy, ForwardingStrategy, QueueVisitor
logger = logging.getLogger(__name__)
@ -73,7 +73,7 @@ def storage(client_name, bucket_name, request):
@pytest.fixture(scope="session", autouse=True)
def docker_compose(sleep_seconds=30):
logger.info(f"Starting docker containers with {COMPOSE_PATH}/docker-compose.yml...")
compose = testcontainers.compose.DockerCompose(COMPOSE_PATH)
compose = testcontainers.compose.DockerCompose(COMPOSE_PATH, compose_file_name="docker-compose.yml")
compose.start()
logger.info(f"Sleeping for {sleep_seconds} seconds to wait for containers to finish startup... ")
time.sleep(sleep_seconds)
@ -81,24 +81,24 @@ def docker_compose(sleep_seconds=30):
compose.stop()
def get_endpoint_from_running_container(container_name, config):
port = config.endpoint.split(":")[-1]
endpoint = f"{docker.get_ip_address(container_name)}:{port}"
return endpoint
# def get_endpoint_from_running_container(container_name, config):
# port = config.endpoint.split(":")[-1]
# endpoint = f"{docker.get_ip_address(container_name)}:{port}"
# return endpoint
def get_pika_connection_params():
params = get_connection_params()
params.host = docker.get_ip_address("rabbitmq")
# params.host = docker.get_ip_address("rabbitmq")
return params
def get_s3_params(s3_backend):
params = CONFIG.storage[s3_backend]
if s3_backend == "minio":
port = params.endpoint.split(":")[-1]
params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}"
# if s3_backend == "minio":
# port = params.endpoint.split(":")[-1]
# params.endpoint = f"{docker.get_ip_address(s3_backend)}:{port}"
return params

21
pyinfra/utils/banner.py Normal file
View File

@ -0,0 +1,21 @@
import logging
from pyinfra.locations import BANNER_FILE
def show_banner():
with open(BANNER_FILE) as f:
banner = "\n" + "".join(f.readlines()) + "\n"
logger = logging.getLogger(__name__)
logger.propagate = False
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
formatter = logging.Formatter("")
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info(banner)

56
pyinfra/utils/docker.py
View File

@ -1,28 +1,28 @@
import docker
from pyinfra.exceptions import NoSuchContainer
def filter_by_name(containers, container_name, exact=False):
def matches(container):
return container_name == container.name if exact else container_name in container.name
containers = filter(matches, containers)
return containers
def get_container(container_name, exact=False):
client = docker.from_env()
containers = client.containers.list()
containers = filter_by_name(containers, container_name, exact=exact)
try:
return next(containers)
except StopIteration:
raise NoSuchContainer(f"No container found for '{container_name}'")
def get_ip_address(container_name, exact=False):
container = get_container(container_name, exact=exact)
ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
return ip_address
# import docker
#
# from pyinfra.exceptions import NoSuchContainer
#
#
# def filter_by_name(containers, container_name, exact=False):
# def matches(container):
# return container_name == container.name if exact else container_name in container.name
#
# containers = filter(matches, containers)
# return containers
#
#
# def get_container(container_name, exact=False):
# client = docker.from_env()
# containers = client.containers.list()
# containers = filter_by_name(containers, container_name, exact=exact)
#
# try:
# return next(containers)
# except StopIteration:
# raise NoSuchContainer(f"No container found for '{container_name}'")
#
#
# def get_ip_address(container_name, exact=False):
# container = get_container(container_name, exact=exact)
# ip_address = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
# return ip_address

13
run_tests.sh
View File

@ -1,16 +1,7 @@
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
docker build -f Dockerfile_tests -t pyinfra-tests:dev .
docker build -f Dockerfile_tests -t pyinfra-tests:dev . --build-arg BASE_ROOT=""
docker tag pyinfra-tests:dev nexus.iqser.com:5001/red/pyinfra-tests:dev
docker push nexus.iqser.com:5001/red/pyinfra-tests:dev
source build_venv/bin/activate
python3 -m pip install --upgrade pip
python3 -m pip install docker-compose
docker-compose -remove-orphans up -d
echo "Sleeping for 30 seconds to wait for dependency containers to finish startup..."
sleep 30s
#docker push nexus.iqser.com:5001/red/pyinfra-tests:dev
rnd=$(date +"%s")
name=pyinfra-tests-${rnd}

9
src/serve.py
View File

@ -4,12 +4,13 @@ from multiprocessing import Process
import requests
from retry import retry
from pyinfra.config import CONFIG, make_art
from pyinfra.config import CONFIG
from pyinfra.exceptions import AnalysisFailure, ConsumerError
from pyinfra.flask import run_probing_webserver, set_up_probing_webserver
from pyinfra.queue.consumer import Consumer
from pyinfra.queue.queue_manager.pika_queue_manager import PikaQueueManager
from pyinfra.storage.storages import get_storage
from pyinfra.utils.banner import show_banner
from pyinfra.visitor import QueueVisitor, StorageStrategy
@ -37,9 +38,9 @@ def make_callback(analysis_endpoint):
def main():
show_banner()
webserver = Process(target=run_probing_webserver, args=(set_up_probing_webserver(),))
logging.info(make_art())
logging.info("Starting webserver...")
webserver.start()
@ -52,11 +53,11 @@ def main():
@retry(ConsumerError, tries=3, delay=5, jitter=(1, 3))
def consume():
consumer = Consumer(visitor, queue_manager)
try:
consumer = Consumer(visitor, queue_manager)
consumer.consume_and_publish()
except Exception as err:
raise ConsumerError from err
raise ConsumerError() from err
try:
consume()