use poetry for dependency management, apply src-layout, run pre-commit

.coveragerc

@@ -8,6 +8,7 @@ omit =
     */__init__.py
     */setup.py
 	*/venv/*
+    */.venv/*
 	*/env/*
 	*/build_venv/*
 	*/build_env/*

.dockerignore

@@ -103,4 +103,4 @@ target/
 *.swp
 */*.swp
 */*/*.swp
-*/*/*/*.swp
+*/*/*/*.swp

.gitignore

@@ -7,4 +7,5 @@ pyinfra.egg-info
 bamboo-specs/target
 .pytest_cache
 /.coverage
-.idea
+.idea
+.vsvode

.pre-commit-config.yaml

@@ -0,0 +1,41 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: ^(docs/|notebooks/|data/|src/secrets/|src/static/|src/templates/|tests)
+default_language_version:
+    python: python3.8
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-yaml
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    - id: flake8
+      args:
+        - "--max-line-length=120"
+
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.10.1
+    hooks:
+    - id: isort
+      args: ["--profile", "black"]
+
+  - repo: https://github.com/psf/black
+    rev: 22.8.0
+    hooks:
+    - id: black
+      # exclude: ^(docs/|notebooks/|data/|src/secrets/)
+      args:
+        - --line-length=120
+
+  - repo: https://github.com/python-poetry/poetry
+    rev: '1.2.0rc1'  # add version here
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+      # - id: poetry-export
+      #   args: ["-f", "requirements.txt", "-o", "requirements.txt"]

.python-version

@@ -0,0 +1 @@
+3.8.13

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "test"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

Makefile

@@ -0,0 +1,58 @@
+.PHONY: \
+	setup-env install install-dev tests \
+	docker-build-run docker-build docker-run \
+	docker-rm docker-rm-container dokcer-rm-image \
+	pre-commit docs sphinx_html sphinx_apidoc
+.DEFAULT_GOAL := run
+
+export DOCKER=docker
+export DOCKERFILE=Dockerfile
+export IMAGE_NAME=pyinfra-image
+export CONTAINER_NAME=pyinfra-container
+export HOST_PORT=9999
+export CONTAINER_PORT=9999
+
+# all commands should be executed in the root dir or the project,
+# specific environments should be deactivated
+
+env-dev:
+	poetry install --dev
+
+install:
+	poetry add $(pkg)
+
+install-dev:
+	poetry add --dev $(pkg)
+
+requirements:
+	poetry export -f requirements.txt --output ./src/requirements.txt
+
+docker-build-run: docker-build docker-run
+
+docker-build:
+	$(DOCKER) build -t $(IMAGE_NAME) -f $(DOCKERFILE) .
+
+docker-run:
+	$(DOCKER) run -it --rm -p $(HOST_PORT):$(CONTAINER_PORT)/tcp --name $(CONTAINER_NAME) $(IMAGE_NAME) python app.py
+
+docker-rm: docker-rm-container docker-rm-image
+
+docker-rm-container:
+	-$(DOCKER) rm $(CONTAINER_NAME)
+
+docker-rm-image:
+	-$(DOCKER) image rm $(IMAGE_NAME)
+
+tests:
+	poetry run pytest ./tests
+
+pre-commit:
+	pre-commit run --all-files
+
+docs: sphinx_apidoc sphinx_html
+
+sphinx_html:
+	poetry run sphinx-build -b html docs/source/ docs/build/html -E -a
+
+sphinx_apidoc:
+	poetry run sphinx-apidoc -o ./docs/source/modules ./src/pyinfra

README.md

@@ -89,7 +89,7 @@ python scripts/manage_minio.py add <MinIO target folder> -d path/to/a/folder/wit
 **Shell 2:** Run pyinfra-server.
 
 ```bash
-python src/serve.py    
+python src/serve.py
 ```
 or as container:
 

bamboo-specs/pom.xml

@@ -37,4 +37,4 @@
 
   <!-- run 'mvn test' to perform offline validation of the plan -->
   <!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
-</project>
+</project>

bamboo-specs/src/main/resources/scripts/create-licence.sh

@@ -16,4 +16,4 @@ then
                     -Dmaven.wagon.http.ssl.allowall=true \
                     -Dmaven.wagon.http.ssl.ignore.validity.dates=true \
                     -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases
-fi
+fi

bamboo-specs/src/main/resources/scripts/git-tag.sh

@@ -6,4 +6,4 @@ then
     echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag
 else
     echo "gitTag=${bamboo_version_tag}" > git.tag
-fi
+fi

bamboo-specs/src/test/java/buildjob/PlanSpecTest.java

@@ -13,4 +13,4 @@ public class PlanSpecTest {
 
         EntityPropertiesBuilders.build(plan);
     }
-}
+}

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/conf.py

@@ -0,0 +1,115 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+sys.path.insert(0, os.path.relpath("../../src"))
+
+import dotenv
+config = dotenv.dotenv_values(dotenv_path="/secrets/.env")
+
+import importlib.metadata
+
+# -- Project information -----------------------------------------------------
+
+DISTRIBUTION_METADATA = importlib.metadata.metadata("pyinfra")
+
+project = "pyinfra"
+copyright = "All rights reserved"
+author = DISTRIBUTION_METADATA["authors"]
+
+# The full version, including alpha/beta/rc tags
+release = DISTRIBUTION_METADATA["version"]
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named "sphinx.ext.*") or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "docxbuilder",
+    "pydata_sphinx_theme",
+    "recommonmark",
+    "readthedocs_ext.readthedocs",
+    "rinoh.frontend.sphinx",
+    "rst2pdf.pdfbuilder",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+# templates_path = ["_templates"]  # specific to some themes, breaks pydata themea
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+source_suffix = [".rst"]
+
+# The master toctree document.
+master_doc = "index"
+
+
+autodoc_mock_imports = ["logging"]
+
+# Additional settings for rhinotype PDF
+rinoh_documents = [
+    (
+        "index",  # top-level file (index.rst)
+        "pyinfra",  # output (target.pdf)
+        "pyinfra" + " Dokumentation",  # document title
+        author
+    )
+]
+
+pdf_documents = [
+    (
+        "index",  # top-level file (index.rst)
+        "pyinfra",  # -> output (target.pdf)
+        "pyinfra" + " Dokumentation",  # document title
+        author
+    )
+]
+
+docx_documents = [
+    (
+        "index",
+        "pyinfra" + ".docx",
+        {
+            "title": "pyinfra" + " Dokumentation",
+            "created": "Francisco Schulz",
+            "subject": "Version: " + release
+        },
+        True
+    ),
+]
+
+docx_update_fields = True
+docx_pagebreak_before_section = 1
+docx_pagebreak_before_table_of_contents = 1
+docx_pagebreak_after_table_of_contents = 1
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "pydata_sphinx_theme"
+
+# # Add any paths that contain custom static files (such as style sheets) here,
+# # relative to this directory. They are copied after the builtin static files,
+# # so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ["_static"]
+
+# html_logo = "_static/logo.jpg"

docs/source/index.rst

@@ -0,0 +1,20 @@
+.. pyinfra documentation master file, created by
+   sphinx-quickstart on Mon Sep 12 12:04:24 2022.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to pyinfra's documentation!
+============================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

notebooks/test.ipynb

@@ -0,0 +1,54 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import pathlib\n",
+    "import os\n",
+    "\n",
+    "os.chdir(\"../src\")\n",
+    "os.getcwd()\n",
+    "\n",
+    "from pyinfra import utils"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.9.12 64-bit ('pypy3.9-7.3.9')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "1b48d261bb3c88480e32a1a672f5ba152bb234aecf443be5b9bd153b58ea3b4e"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

pyproject.toml

@@ -0,0 +1,44 @@
+[tool.poetry]
+name = "pyinfra"
+version = "1.1.0"
+description = "python package that is supposed to handle networking and connections for model packages that are used for predictions in the RED project"
+authors = ["Frankcisco Schulz"]
+license = "All rights reserved"
+packages = [
+    { include = "pyinfra", from = "src" }
+]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+python-dotenv = "^0.19.2"
+pika = "^1.3.0"
+retry = "^0.9.2"
+minio = "^7.1.11"
+azure-core = "^1.25.1"
+azure-storage-blob = "^12.13.1"
+testcontainers = "^3.6.1"
+funcy = "^1.17"
+
+[tool.poetry.dev-dependencies]
+pylint = "^2.12.1"
+black = "^21.11b1"
+pytest = "^6.2.5"
+pytest-mock = "^3.6.1"
+ipykernel = "^6.6.0"
+pre-commit = "^2.16.0"
+isort = "^5.10.1"
+Sphinx = "^4.4.0"
+recommonmark = "^0.7.1"
+readthedocs-sphinx-ext = "^2.1.4"
+rinohtype = "^0.5.3"
+rst2pdf = "^0.99"
+pydata-sphinx-theme = "^0.8.0"
+docxbuilder = "^1.2.0"
+
+[tool.pytest.ini_options]
+log_cli = 1
+log_cli_level = "DEBUG"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

pytest.ini

@@ -1,4 +1,3 @@
 [pytest]
 log_cli = 1
 log_cli_level = DEBUG
-

requirements.txt

@@ -1,9 +0,0 @@
-pika==1.2.0
-retry==0.9.2
-minio==7.1.3
-azure-core==1.22.1
-azure-storage-blob==12.9.0
-testcontainers==3.4.2
-docker-compose==1.29.2
-pytest~=7.0.1
-funcy==1.17

setup.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-
-from distutils.core import setup
-
-setup(
-    name="pyinfra",
-    version="0.0.1",
-    description="",
-    author="",
-    author_email="",
-    url="",
-    packages=["pyinfra"],
-)

src/pyinfra/k8s_probes/startup.py

@@ -1,6 +1,6 @@
 import logging
-from pathlib import Path
 import sys
+from pathlib import Path
 
 from pyinfra.queue.queue_manager import token_file_name
 
@@ -29,7 +29,7 @@ def check_token_file():
         return False
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     if check_token_file():
         sys.exit(0)
     else:

src/pyinfra/queue/queue_manager.py

@@ -2,8 +2,8 @@ import atexit
 import json
 import logging
 import signal
-from typing import Callable
 from pathlib import Path
+from typing import Callable
 
 import pika
 import pika.exceptions
@@ -91,9 +91,7 @@ class QueueManager(object):
             self.logger.info(f"Registered with consumer-tag: {self._consumer_token}")
             self._channel.start_consuming()
         except Exception:
-            self.logger.warning(
-                "An unexpected exception occurred while consuming messages.  Consuming will stop."
-            )
+            self.logger.warning("An unexpected exception occurred while consuming messages.  Consuming will stop.")
             raise
         finally:
             self.stop_consuming()
@@ -117,8 +115,9 @@ class QueueManager(object):
             # Requeueing will be handled by the dead-letter-exchange.
             # This prevents endless retries on messages that are impossible to process.
             if frame.redelivered:
-                self.logger.info(f"Aborting message processing for delivery_tag {frame.delivery_tag} "
-                                 f"due to it being redelivered")
+                self.logger.info(
+                    f"Aborting message processing for delivery_tag {frame.delivery_tag} " f"due to it being redelivered"
+                )
                 self._channel.basic_nack(frame.delivery_tag, requeue=False)
                 return
 
@@ -129,8 +128,10 @@ class QueueManager(object):
                 should_publish_result, callback_result = process_message_callback(unpacked_message_body)
 
                 if should_publish_result:
-                    self.logger.info(f"Processed message with delivery_tag {frame.delivery_tag}, "
-                                     f"publishing result to result-queue")
+                    self.logger.info(
+                        f"Processed message with delivery_tag {frame.delivery_tag}, "
+                        f"publishing result to result-queue"
+                    )
                     self._channel.basic_publish("", self._output_queue, json.dumps(callback_result).encode())
 
                     self.logger.info(

src/pyinfra/storage/adapters/azure.py

@@ -2,7 +2,7 @@ import logging
 from itertools import repeat
 from operator import attrgetter
 
-from azure.storage.blob import ContainerClient, BlobServiceClient
+from azure.storage.blob import BlobServiceClient, ContainerClient
 from retry import retry
 
 from pyinfra.config import Config, get_config

src/pyinfra/storage/adapters/s3.py

@@ -10,7 +10,6 @@ from retry import retry
 
 from pyinfra.config import Config, get_config
 
-
 CONFIG = get_config()
 logger = logging.getLogger(CONFIG.logging_level_root)
 

src/pyinfra/storage/storage.py

@@ -1,6 +1,4 @@
-import logging
-
-from pyinfra.config import get_config, Config
+from pyinfra.config import Config
 from pyinfra.storage.adapters.azure import get_azure_storage
 from pyinfra.storage.adapters.s3 import get_s3_storage
 

tests/conftest.py

@@ -0,0 +1,35 @@
+import os
+import pathlib
+import sys
+
+MODULE_DIR = os.path.abspath(pathlib.Path("./src"))
+sys.path.append(MODULE_DIR)
+os.chdir(MODULE_DIR)
+
+import logging
+
+import dotenv
+import pytest
+import yaml
+
+logger = logging.getLogger(__name__)
+
+logger.info(os.getcwd())
+
+
+@pytest.fixture
+def test_param_dict():
+    test_param_dict = yaml.load(open(pathlib.Path("../tests/test_params.yml"), "r"), yaml.FullLoader)
+    return test_param_dict
+
+
+@pytest.fixture
+def secrets():
+    return dotenv.dotenv_values(dotenv_path="/configs/.env")
+
+
+@pytest.fixture
+def current_version():
+    with open(pathlib.Path("./VERSION")) as f:
+        v = f.read()
+    return v

tests/test_generic.py

@@ -0,0 +1,5 @@
+from pyinfra import __version__
+
+
+def test_version(current_version):
+    assert __version__ == current_version

tests/test_params.yml

@@ -0,0 +1,25 @@
+url: "https://send.help"
+
+endpoint: "/predict"
+
+header: headless
+
+response: |-
+  {
+    "meta": {
+      ...
+    },
+    "data": [
+      ...
+    ]
+  }
+
+request: |-
+  {
+    "meta": {
+      ...
+    },
+    "data": [
+      ...
+    ]
+  }