Merge branch 'chore/nexus-package-registry' into 'master'

RES-914: move package registry to nexus

See merge request knecon/research/pyinfra!106

.gitlab-ci.yml

@@ -1,49 +1,23 @@
+# CI for services, check gitlab repo for python package CI
 include:
   - project: "Gitlab/gitlab"
-    ref: 0.3.0
-    file: "/ci-templates/research/python_pkg_venv_test_build_release_gitlab-ci.yml"
-
-default:
-  image: python:3.10
+    ref: main
+    file: "/ci-templates/research/python_pkg-test-build-release.gitlab-ci.yml"
 
+# set project variables here
 variables:
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  GITLAB_PYPI_URL: https://gitlab.knecon.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi
-  PYPI_REGISTRY_RESEARCH: https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi
-  POETRY_SOURCE_REF_RESEARCH: gitlab-research
-  PYPI_REGISTRY_RED: https://gitlab.knecon.com/api/v4/groups/12/-/packages/pypi
-  POETRY_SOURCE_REF_RED: gitlab-red
-  PYPI_REGISTRY_FFORESIGHT: https://gitlab.knecon.com/api/v4/groups/269/-/packages/pypi
-  POETRY_SOURCE_REF_FFORESIGHT: gitlab-fforesight
-  # POETRY_HOME: /opt/poetry
+  NEXUS_PROJECT_DIR: research # subfolder in Nexus docker-gin where your container will be stored
+  IMAGENAME: $CI_PROJECT_NAME # if the project URL is gitlab.example.com/group-name/project-1, CI_PROJECT_NAME is project-1
+  REPORTS_DIR: reports
+  FF_USE_FASTZIP: "true" # enable fastzip - a faster zip implementation that also supports level configuration.
+  ARTIFACT_COMPRESSION_LEVEL: default # can also be set to fastest, fast, slow and slowest. If just enabling fastzip is not enough try setting this to fastest or fast.
+  CACHE_COMPRESSION_LEVEL: default # same as above, but for caches
+  # TRANSFER_METER_FREQUENCY: 5s # will display transfer progress every 5 seconds for artifacts and remote caches. For debugging purposes.
 
-setup-poetry-venv:
-  stage: setup
-  script:
-    - env # check env vars
-    # install poetry & return versions
-    - pip install --upgrade pip
-    - pip -V
-    - python -V
-    - pip install poetry
-    - poetry -V
-    # configure poetry
-    - poetry config installer.max-workers 10
-    - poetry config virtualenvs.in-project true
-    - poetry config repositories.${POETRY_SOURCE_REF_RESEARCH} ${PYPI_REGISTRY_RESEARCH}
-    - poetry config http-basic.${POETRY_SOURCE_REF_RESEARCH} ${CI_REGISTRY_USER} ${CI_JOB_TOKEN}
-    - poetry config repositories.${POETRY_SOURCE_REF_RED} ${PYPI_REGISTRY_RED}
-    - poetry config http-basic.${POETRY_SOURCE_REF_RED} ${CI_REGISTRY_USER} ${CI_JOB_TOKEN}
-    - poetry config repositories.${POETRY_SOURCE_REF_FFORESIGHT} ${PYPI_REGISTRY_FFORESIGHT}
-    - poetry config http-basic.${POETRY_SOURCE_REF_FFORESIGHT} ${CI_REGISTRY_USER} ${CI_JOB_TOKEN}
-    # create and activate venv
-    - poetry env use $(which python)
-    - source .venv/bin/activate
-    - python -m ensurepip
-    - env # check env vars again
-    # install from poetry.lock file
-    - poetry install --all-extras -vvv
 
-run-tests:
-  script:
-    - echo "Disabled until we have an automated way to run docker compose before tests."
+############
+# UNIT TESTS
+unit-tests:
+  variables:
+    ###### UPDATE/EDIT ######
+    UNIT_TEST_DIR: "tests/unit_test"

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
   python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -39,7 +39,7 @@ repos:
           - --profile black
 
   - repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
         # exclude: ^(docs/|notebooks/|data/|src/secrets/)
@@ -47,7 +47,7 @@ repos:
           - --line-length=120
 
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v3.2.0
+    rev: v3.6.0
     hooks:
       - id: conventional-pre-commit
         pass_filenames: false

README.md

@@ -73,6 +73,17 @@ the [complete example](pyinfra/examples.py).
 | TRACING\_\_OPENTELEMETRY\_\_EXPORTER       | tracing.opentelemetry.exporter          | Name of exporter                                                                                                                                                                                                                                                        |
 | KUBERNETES\_\_POD_NAME                     | kubernetes.pod_name                     | Service pod name                                                                                                                                                                                                                                                        |
 
+## Setup
+**IMPORTANT** you need to set the following environment variables before running the setup script:
+- ``$NEXUS_USER`` your Nexus user (usually equal to firstname.lastname@knecon.com)
+- ``$NEXUS_PASSWORD`` your Nexus password (usually equal to your Azure Login)
+
+```shell
+# create venv and activate it
+source ./scripts/setup/devenvsetup.sh {{ cookiecutter.python_version }} $NEXUS_USER $NEXUS_PASSWORD
+source .venv/bin/activate
+```
+
 ### OpenTelemetry
 
 Open telemetry (vis its Python SDK) is set up to be as unobtrusive as possible; for typical use cases it can be

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyinfra"
-version = "3.3.5"
+version = "3.4.0"
 description = ""
 authors = ["Team Research <research@knecon.com>"]
 license = "All rights reseverd"
@@ -18,28 +18,29 @@ azure-storage-blob = "^12.13"
 # misc utils
 funcy = "^2"
 pycryptodome = "^3.19"
-# research shared packages
-kn-utils = { version = "^0.3.2", source = "gitlab-research" }
 fastapi = "^0.109.0"
 uvicorn = "^0.26.0"
+
+[tool.poetry.group.internal.dependencies]
+kn-utils = { version = "0.4.0", source = "nexus" }
+
 # [tool.poetry.group.telemetry.dependencies]
-opentelemetry-instrumentation-pika = "^0.46b0"
-opentelemetry-exporter-otlp = "^1.25.0"
-opentelemetry-instrumentation = "^0.46b0"
-opentelemetry-api = "^1.25.0"
-opentelemetry-sdk = "^1.25.0"
-opentelemetry-exporter-otlp-proto-http = "^1.25.0"
-opentelemetry-instrumentation-flask = "^0.46b0"
-opentelemetry-instrumentation-requests = "^0.46b0"
-opentelemetry-instrumentation-fastapi = "^0.46b0"
+# We set all opentelemetry dependencies to lower bound because the image classification service depends on a protobuf version <4, but does not use proto files.
+# Therefore, we allow latest possible protobuf version in the services which use proto files. As soon as the dependency issue is fixed set this to the latest possible opentelemetry version
+opentelemetry-instrumentation-pika = ">=0.46b0,<0.50"
+opentelemetry-exporter-otlp = ">=1.25.0,<1.29"
+opentelemetry-instrumentation = ">=0.46b0,<0.50"
+opentelemetry-api = ">=1.25.0,<1.29"
+opentelemetry-sdk = ">=1.25.0,<1.29"
+opentelemetry-exporter-otlp-proto-http = ">=1.25.0,<1.29"
+opentelemetry-instrumentation-flask = ">=0.46b0,<0.50"
+opentelemetry-instrumentation-requests = ">=0.46b0,<0.50"
+opentelemetry-instrumentation-fastapi = ">=0.46b0,<0.50"
+opentelemetry-instrumentation-aio-pika = ">=0.46b0,<0.50"
 wcwidth = "<=0.2.12"
 azure-monitor-opentelemetry = "^1.6.0"
-# We set protobuf to this range because the image classification service depends on a protobuf version <4, but does not use proto files.
-# Therefore, we allow latest possible protobuf version in the services which use proto files. As soon as the dependency issue is fixed set this to the latest possible protobuf version
-protobuf = ">=3.20 <5.0.0"
 aio-pika = "^9.4.2"
 aiohttp = "^3.9.5"
-opentelemetry-instrumentation-aio-pika = "0.46b0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7"
@@ -53,6 +54,7 @@ cyclonedx-bom = "^4.1.1"
 dvc = "^3.51.2"
 dvc-azure = "^3.1.0"
 deepdiff = "^7.0.1"
+pytest-cov = "^5.0.0"
 
 [tool.pytest.ini_options]
 minversion = "6.0"
@@ -87,12 +89,13 @@ disable = [
 docstring-min-length = 3
 
 [[tool.poetry.source]]
-name = "PyPI"
+name = "pypi-proxy"
+url = "https://nexus.knecon.com/repository/pypi-proxy/simple"
 priority = "primary"
 
 [[tool.poetry.source]]
-name = "gitlab-research"
-url = "https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi/simple"
+name = "nexus"
+url = "https://nexus.knecon.com/repository/python/simple"
 priority = "explicit"
 
 [build-system]

scripts/setup/devenvsetup.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+python_version=$1
+nexus_user=$2
+nexus_password=$3
+
+# cookiecutter https://gitlab.knecon.com/knecon/research/template-python-project.git --checkout master
+# latest_dir=$(ls -td -- */ | head -n 1)  # should be the dir cookiecutter just created
+
+# cd $latest_dir
+
+pyenv install $python_version
+pyenv local $python_version
+pyenv shell $python_version
+
+# install poetry globally (PREFERRED), only need to install it once
+# curl -sSL https://install.python-poetry.org | python3 -
+
+# remember to update poetry once in a while
+poetry self update
+
+# install poetry in current python environment, can lead to multiple instances of poetry being installed on one system (DISPREFERRED)
+# pip install --upgrade pip
+# pip install poetry
+
+poetry config virtualenvs.in-project true
+poetry config installer.max-workers 10
+poetry config repositories.pypi-proxy "https://nexus.knecon.com/repository/pypi-proxy/simple"
+poetry config http-basic.pypi-proxy ${nexus_user} ${nexus_password}
+poetry config repositories.nexus https://nexus.knecon.com/repository/python/simple
+poetry config http-basic.nexus ${nexus_user} ${nexus_password}
+
+poetry env use $(pyenv which python)
+poetry install --with=dev
+poetry update
+
+source .venv/bin/activate
+
+pre-commit install
+pre-commit autoupdate