update project structure

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: ^(docs/|notebooks/|data/|src/secrets/|src/static/|src/templates/|tests)
+default_language_version:
+  python: python3.8
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+
+  - repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+      - id: black
+        # exclude: ^(docs/|notebooks/|data/|src/secrets/)
+        args:
+          - --line-length=120

.python-version

@@ -0,0 +1 @@
+3.8.13

Dockerfile

@@ -1,27 +1,43 @@
-FROM image-prediction-base
+FROM python:3.8 as builder
+ARG SSH_KEY
+# ARG SSH_KEY_PUB
+ENV PYTHONUNBUFFERED=true
+WORKDIR /app
 
-WORKDIR /app/service
+ENV POETRY_HOME=/opt/poetry
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV PATH="$POETRY_HOME/bin:$PATH"
 
-COPY src src
+RUN mkdir /root/.ssh/ && \
-COPY incl/pyinfra incl/pyinfra
+    chmod 700 /root/.ssh/ && \
-COPY incl/pdf2image incl/pdf2image
+    echo "${SSH_KEY}" > /root/.ssh/id_rsa && \
-COPY data data
+    chmod 600 /root/.ssh/id_rsa && \
-COPY image_prediction image_prediction
+    ssh-keyscan -p 2222 git.iqser.com >> /root/.ssh/known_hosts
-COPY setup.py setup.py
+# echo "${SSH_KEY_PUB}" > /root/.ssh/id_rsa.pub && \
-COPY requirements.txt requirements.txt
+# chmod 600 /root/.ssh/id_rsa.pub && \
-COPY config.yaml config.yaml
-COPY banner.txt banner.txt
 
-# Install dependencies differing from base image.
+RUN curl -sSL https://install.python-poetry.org | python3 -
-RUN python3 -m pip install -r requirements.txt
-RUN python3 -m pip install -r incl/pyinfra/requirements.txt
-RUN python3 -m pip install -r incl/pdf2image/requirements.txt
 
-RUN python3 -m pip install -e .
+COPY pyproject.toml poetry.lock ./
-RUN python3 -m pip install -e incl/pyinfra
+RUN poetry config virtualenvs.create false && \
-RUN python3 -m pip install -e incl/pdf2image
+    poetry install --without=dev -vvv --no-interaction --no-root
+
+FROM python:3.8-bullseye as prod
+ENV PYTHONUNBUFFERED=true
+WORKDIR /app
+ENV PATH="/app/.venv/bin:$PATH"
+COPY --from=builder /app /app
+
+COPY ./src ./
+USER 1001
 
 EXPOSE 5000
 EXPOSE 8080
 
 CMD ["python3", "src/serve.py"]
+
+# TESTS
+# RUN apt update --yes
+# RUN apt install vim --yes
+# RUN apt install poppler-utils --yes
+# CMD coverage run -m pytest test/ --tb=native -q -s -vvv -x && coverage combine && coverage report -m && coverage xml

Dockerfile_base

@@ -1,25 +0,0 @@
-FROM python:3.8 as builder1
-
-# Use a virtual environment.
-RUN python -m venv /app/venv
-ENV PATH="/app/venv/bin:$PATH"
-
-# Upgrade pip.
-RUN python -m pip install --upgrade pip
-
-# Make a directory for the service files and copy the service repo into the container.
-WORKDIR /app/service
-COPY ./requirements.txt ./requirements.txt
-
-# Install dependencies.
-RUN python3 -m pip install -r requirements.txt
-
-# Make a new container and copy all relevant files over to filter out temporary files
-# produced during setup to reduce the final container's size.
-FROM python:3.8
-
-WORKDIR /app/
-COPY --from=builder1  /app .
-ENV PATH="/app/venv/bin:$PATH"
-
-WORKDIR /app/service

Dockerfile_tests

@@ -1,31 +0,0 @@
-ARG BASE_ROOT="nexus.iqser.com:5001/red/"
-ARG VERSION_TAG="dev"
-
-FROM ${BASE_ROOT}image-prediction:${VERSION_TAG}
-
-WORKDIR /app/service
-
-COPY src src
-COPY incl/pyinfra incl/pyinfra
-COPY incl/pdf2image incl/pdf2image
-COPY data data
-COPY image_prediction image_prediction
-COPY setup.py setup.py
-COPY requirements.txt requirements.txt
-COPY config.yaml config.yaml
-COPY banner.txt banner.txt
-
-# Install module & dependencies
-RUN python3 -m pip install -r requirements.txt
-RUN python3 -m pip install -r incl/pyinfra/requirements.txt
-RUN python3 -m pip install -r incl/pdf2image/requirements.txt
-
-RUN python3 -m pip install -e .
-RUN python3 -m pip install -e incl/pyinfra
-RUN python3 -m pip install -e incl/pdf2image
-
-RUN apt update --yes
-RUN apt install vim --yes
-RUN apt install poppler-utils --yes
-
-CMD coverage run -m pytest test/ --tb=native -q -s -vvv -x && coverage combine && coverage report -m && coverage xml

Makefile

@@ -0,0 +1,88 @@
+.PHONY: \
+	poetry in-project-venv dev-env use-env install install-dev tests \
+	update-version sync-version-with-git \
+	docker docker-build-run docker-build docker-run \
+	docker-rm docker-rm-container docker-rm-image \
+	pre-commit get-licenses prep-commit \
+	docs sphinx_html sphinx_apidoc
+.DEFAULT_GOAL := run
+
+export DOCKER=docker
+export DOCKERFILE=Dockerfile
+export IMAGE_NAME=imgage-prediction-image
+export CONTAINER_NAME=imgage-prediction-container
+export HOST_PORT=9999
+export CONTAINER_PORT=9999
+export PYTHON_VERSION=python3.8
+export SSH_KEY=$(cat ~/.ssh/id_ecdsa)
+
+# all commands should be executed in the root dir or the project,
+# specific environments should be deactivated
+
+poetry: in-project-venv use-env dev-env
+
+in-project-venv:
+	poetry config virtualenvs.in-project true
+
+use-env:
+	poetry env use ${PYTHON_VERSION}
+
+dev-env:
+	poetry install --with dev
+
+install:
+	poetry add $(pkg)
+
+install-dev:
+	poetry add --dev $(pkg)
+
+requirements:
+	poetry export --without-hashes --output ./src/requirements.txt
+
+update-version:
+	poetry version prerelease
+
+sync-version-with-git:
+	git pull -p && poetry version $(git rev-list --tags --max-count=1 | git describe --tags --abbrev=0)
+
+docker: docker-rm docker-build-run
+
+docker-build-run: docker-build docker-run
+
+docker-build:
+	$(DOCKER) build \
+	--no-cache --progress=plain \
+	--build-arg SSH_KEY=$(SSH_KEY) \
+	-t $(IMAGE_NAME) \
+	-f $(DOCKERFILE) .
+
+docker-run:
+	$(DOCKER) run -it --rm -p $(HOST_PORT):$(CONTAINER_PORT)/tcp --name $(CONTAINER_NAME) $(IMAGE_NAME) python ./src/serve.py
+
+docker-rm: docker-rm-container docker-rm-image
+
+docker-rm-container:
+	-$(DOCKER) rm $(CONTAINER_NAME)
+
+docker-rm-image:
+	-$(DOCKER) image rm $(IMAGE_NAME)
+
+tests:
+	poetry run pytest ./tests
+
+prep-commit:
+	docs get-license sync-version-with-git update-version pre-commit
+
+pre-commit:
+	pre-commit run --all-files
+
+get-licenses:
+	pip-licenses --format=json --order=license --with-urls > pkg-licenses.json
+
+docs: sphinx_apidoc sphinx_html
+
+sphinx_html:
+	poetry run sphinx-build -b html docs/source/ docs/build/html -E -a
+
+sphinx_apidoc:
+	poetry run sphinx-apidoc -o ./docs/source/modules ./src/rule_engine

README.md

@@ -34,7 +34,7 @@ python scripts/pyinfra_mock.py /path/to/a/pdf
 Run for example this command to execute all tests and get a coverage report:
 
 ```bash
-coverage run -m pytest test --tb=native -q -s -vvv -x && coverage combine && coverage report -m
+coverage run -m pytest tests --tb=native -q -s -vvv -x && coverage combine && coverage report -m
 ```
 
 After having built the service container as specified above, you can also run tests in a container as follows:
@@ -126,15 +126,15 @@ An image metadata record (entry in `"data"` field of a response body) looks like
 A configuration file is located under `config.yaml`. All relevant variables can be configured via
 exporting environment variables.
 
-| __Environment Variable__           | Default                            | Description                                                                            |
+| __Environment Variable__ | Default                            | Description                                                    |
-|------------------------------------|------------------------------------|----------------------------------------------------------------------------------------|
+| ------------------------ | ---------------------------------- | -------------------------------------------------------------- |
-| __LOGGING_LEVEL_ROOT__             | "INFO"                             | Logging level for log file messages                                                    |
+| __LOGGING_LEVEL_ROOT__   | "INFO"                             | Logging level for log file messages                            |
-| __VERBOSE__                        | *true*                             | Service prints document processing progress to stdout                                  |
+| __VERBOSE__              | *true*                             | Service prints document processing progress to stdout          |
-| __BATCH_SIZE__                     | 16                                 | Number of images in memory simultaneously per service instance                         |
+| __BATCH_SIZE__           | 16                                 | Number of images in memory simultaneously per service instance |
-| __RUN_ID__                         | "fabfb1f192c745369b88cab34471aba7" | The ID of the mlflow run to load the image classifier from                             |
+| __RUN_ID__               | "fabfb1f192c745369b88cab34471aba7" | The ID of the mlflow run to load the image classifier from     |
-| __MIN_REL_IMAGE_SIZE__             | 0.05                               | Minimally permissible image size to page size ratio                                    |
+| __MIN_REL_IMAGE_SIZE__   | 0.05                               | Minimally permissible image size to page size ratio            |
-| __MAX_REL_IMAGE_SIZE__             | 0.75                               | Maximally permissible image size to page size ratio                                    |
+| __MAX_REL_IMAGE_SIZE__   | 0.75                               | Maximally permissible image size to page size ratio            |
-| __MIN_IMAGE_FORMAT__               | 0.1                                | Minimally permissible image width to height ratio                                      |
+| __MIN_IMAGE_FORMAT__     | 0.1                                | Minimally permissible image width to height ratio              |
-| __MAX_IMAGE_FORMAT__               | 10                                 | Maximally permissible image width to height ratio                                      |
+| __MAX_IMAGE_FORMAT__     | 10                                 | Maximally permissible image width to height ratio              |
 
 See also: https://git.iqser.com/projects/RED/repos/helm/browse/redaction/templates/image-service-v2

bamboo-specs/src/main/java/buildjob/PlanSpec.java

@@ -33,21 +33,22 @@ import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
 
 /**
  * Plan configuration for Bamboo.
- * Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
+ * Learn more on: <a href=
+ *
+ * "https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
  */
 @BambooSpec
 public class PlanSpec {
 
     private static final String SERVICE_NAME = "image-prediction";
-    private static final String SERVICE_NAME_BASE = "image-prediction-base";
 
-    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-","").replaceAll("_","");
+    private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "").replaceAll("_", "");
 
     /**
      * Run main to publish plan on Bamboo
      */
-    public static void main(final String[] args) throws Exception {
+    public  static void main(final String[] args) throws Exception {
-        //By default credentials are read from the '.credentials' file.
+        // By default credentials are read from the '.credentials' file.
         BambooServer bambooServer = new BambooServer("http://localhost:8085");
 
         Plan plan = new PlanSpec().createDockerBuildPlan();
@@ -63,10 +64,18 @@ public class PlanSpec {
 
     private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
         Permissions permission = new Permissions()
-                .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
+
-                .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN,
-                .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+
-                .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
+                        PermissionType.CLONE, PermissionType.BUILD)
+
+                .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.
+                        LONE,
+                        PermissionType.BUILD)
+                .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE,
+                        PermissionType.BUILD)
+                .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE,
+                        PermissionType.BUILD)
                 .loggedInUserPermissions(PermissionType.VIEW)
                 .anonymousUserPermissionView();
         return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
@@ -79,102 +88,110 @@ public class PlanSpec {
     }
 
     public Plan createDockerBuildPlan() {
-    return new Plan(
+        return new Plan(
-            project(),
+                project(),
-            SERVICE_NAME, new BambooKey(SERVICE_KEY))
+                SERVICE_NAME, new BambooKey(SERVICE_KEY))
-            .description("Docker build for image-prediction.")
+                .description("Docker build for image-prediction.")
-            .stages(
+                .stages(
-            new Stage("Build Stage")
+                        new Stage("Build Stage")
-              .jobs(
+                                .jobs(
-                new Job("Build Job", new BambooKey("BUILD"))
+                                        new Job("Build Job", new BambooKey("BUILD"))
-                  .tasks(
+                                                .tasks(
-                    new CleanWorkingDirectoryTask()
+                                                        new CleanWorkingDirectoryTask()
-                        .description("Clean working directory.")
+                                                                .description("Clean working directory.")
-                        .enabled(true),
+                                                                .enabled(true),
-                    new VcsCheckoutTask()
+                                                        new VcsCheckoutTask()
-                        .description("Checkout default repository.")
+                                                                .description("Checkout default repository.")
-                        .checkoutItems(new CheckoutItem().defaultRepository()),
+                                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                    new ScriptTask()
+                                                        new ScriptTask()
-                        .description("Set config and keys.")
+                                                                .description("Set config and keys.")
-                        .location(Location.FILE)
+                                                                .location(Location.FILE)
-                        .fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
+                                                                .fileFromPath(
-                    new ScriptTask()
+                                                                        "bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
-                        .description("Build Docker container.")
+                                                        new ScriptTask()
-                        .location(Location.FILE)
+                                                                .description("Build Docker container.")
-                        .fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh")
+                                                                .location(Location.FILE)
-                        .argument(SERVICE_NAME + " " + SERVICE_NAME_BASE),
+                                                                .fileFromPath(
-                    new InjectVariablesTask()
+                                                                        "bamboo-specs/src/main/resources/scripts/docker-build.sh")
-                        .description("Inject git tag.")
+                                                                .argument(SERVICE_NAME),
-                        .path("git.tag")
+                                                        new InjectVariablesTask()
-                        .namespace("g")
+                                                                .description("Inject git tag.")
-                        .scope(InjectVariablesScope.LOCAL),
+                                                                .path("git.tag")
-                    new VcsTagTask()
+                                                                .namespace("g")
-                        .description("${bamboo.g.gitTag}")
+                                                                .scope(InjectVariablesScope.LOCAL),
-                        .tagName("${bamboo.g.gitTag}")
+                                                        new VcsTagTask()
-                        .defaultRepository())
+                                                                .description("${bamboo.g.gitTag}")
-                .dockerConfiguration(
+                                                                .tagName("${bamboo.g.gitTag}")
-                    new DockerConfiguration()
+                                                                .defaultRepository())
-                      .image("nexus.iqser.com:5001/infra/release_build:4.5.0")
+                                                .dockerConfiguration(
-                      .volume("/var/run/docker.sock", "/var/run/docker.sock")),
+                                                        new DockerConfiguration()
-                new Job("Licence Job", new BambooKey("LICENCE"))
+                                                                .image("nexus.iqser.com:5001/infra/release_build:4.5.0")
-                  .enabled(false)
+                                                                .volume("/var/run/docker.sock",
-                  .tasks(
+                                                                        "/var/run/docker.sock")),
-                    new VcsCheckoutTask()
+                                        new Job("Licence Job", new BambooKey("LICENCE"))
-                        .description("Checkout default repository.")
+                                                .enabled(false)
-                        .checkoutItems(new CheckoutItem().defaultRepository()),
+                                                .tasks(
-                    new ScriptTask()
+                                                        new VcsCheckoutTask()
-                        .description("Build licence.")
+                                                                .description("Checkout default repository.")
-                        .location(Location.FILE)
+                                                                .checkoutItems(new CheckoutItem().defaultRepository()),
-                        .fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh"))
+                                                        new ScriptTask()
-                  .dockerConfiguration(
+                                                                .description("Build licence.")
-                    new DockerConfiguration()
+                                                                .location(Location.FILE)
-                        .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
+                                                                .fileFromPath(
-                        .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
+                                                                        "bamboo-specs/src/main/resources/scripts/create-licence.sh"))
-                        .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+                                                .dockerConfiguration(
-            .linkedRepositories("RR / " + SERVICE_NAME)
+                                                        new DockerConfiguration()
-            .linkedRepositories("RR / redai_image")
+                                                                .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
-            .triggers(
+                                                                .volume("/etc/maven/settings.xml",
-                    new BitbucketServerTrigger())
+                                                                        "/usr/share/maven/ref/settings.xml")
-            .planBranchManagement(
+                                                                .volume("/var/run/docker.sock",
-                new PlanBranchManagement()
+                                                                        "/var/run/docker.sock"))))
-                    .createForVcsBranch()
+                .linkedRepositories("RR / " + SERVICE_NAME)
-                    .delete(
+                .linkedRepositories("RR / redai_image")
-                        new BranchCleanup()
+                .triggers(
-                            .whenInactiveInRepositoryAfterDays(14))
-                    .notificationForCommitters());
-    }
 
-    public Plan createSecBuild() {
+                        new BitbucketServerTrigger())
-        return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
+                        nBranchManagement(
-                .stages(new Stage("Default Stage").jobs(
+                                  new P
-                    new Job("Sonar Job", new BambooKey("SONAR"))
+                                                .createForVcsBranch()
-                      .tasks(
+                                                    .delete(
-                        new CleanWorkingDirectoryTask()
+                                                            new
-                            .description("Clean working directory.")
+
-                            .enabled(true),
+                                                    .notificationForCommitters());
-                        new VcsCheckoutTask()
+
-                            .description("Checkout default repository.")
+
-                            .checkoutItems(new CheckoutItem().defaultRepository()),
+                                                ld() {
-                        new ScriptTask()
+                                                ect(), SERVICE_NAME + "-
-                            .description("Set config and keys.")
+                                                ("Security Analysis Plan")
-                            .location(Location.FILE)
+                                        new Stage("Defau
-                            .fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
+                                                Job("Sonar Job", new BambooKey("SON
-                        new ScriptTask()
+                                                    .tasks(
-                            .description("Run Sonarqube scan.")
+                                                            new CleanWorkingDirectoryTask()
-                            .location(Location.FILE)
+                                                                    .des
-                            .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh")
+
-                            .argument(SERVICE_NAME))
+                                                      new VcsChec
-                      .dockerConfiguration(
+                                                                    .description("Checkout default repos
-                          new DockerConfiguration()
+                                                                    .checkoutItems(new CheckoutItem().defaultRepository()),
-                            .image("nexus.iqser.com:5001/infra/release_build:4.2.0")
+                                        new ScriptTask()
-                            .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
+
+
+                                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
+
+
+                                                        .location(Locati
+                                                        .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh")
+                                                .argument(SERVICE_NAME))
+                                .dockerConfiguration(
+                                        new DockerConfiguration()
+                                                .image("nexus.iqser.com:5001/infra/release_build:4.2.0")
+                                                .volume("/var/run/docker.sock", "/var/run/docker.sock"))))
                 .linkedRepositories("RR / " + SERVICE_NAME)
                 .triggers(
-                    new ScheduledTrigger()
+                        new ScheduledTrigger()
-                        .scheduleOnceDaily(LocalTime.of(23, 00)))
+                                .scheduleOnceDaily(LocalTime.of(23, 00)))
                 .planBranchManagement(
-                    new PlanBranchManagement()
+                        new PlanBranchManagement()
-                        .createForVcsBranchMatching("release.*")
+                                .createForVcsBranchMatching("release.*")
-                        .notificationForCommitters());
+                                .notificationForCommitters());
     }
 }

bamboo-specs/src/main/resources/scripts/docker-build.sh

@@ -2,55 +2,97 @@
 set -e
 
 SERVICE_NAME=$1
-SERVICE_NAME_BASE=$2
+project_name="image-prediction"
-if [[ "$bamboo_planRepository_branchName" == "master" ]]
+HOST="nexus.iqser.com"
-then
+PORT="5001"
-    branchVersion=$(cat version.yaml | grep -Eo "version: .*" | sed -s 's|version: \(.*\)\..*\..*|\1|g')
-    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
-    newVersion="$(semver $latestVersion -p -i minor)"
-    echo "new release on master with version $newVersion"
-elif [[ "$bamboo_planRepository_branchName" == release* ]]
-then
-    branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
-    latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
-    newVersion="$(semver $latestVersion -p -i patch)"
-    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
-elif [[ "${bamboo_version_tag}" != "dev" ]]
-then
-    newVersion="${bamboo_version_tag}"
-    echo "new special version bild with $newVersion"
-else
-    newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
-    echo "gitTag=${newVersion}" > git.tag
-    echo "dev build with tag ${newVersion}"
-    python3 -m venv build_venv
-    source build_venv/bin/activate
-    python3 -m pip install --upgrade pip
 
-    pip install dvc
+mkdir -p ~/.ssh
-    pip install 'dvc[ssh]'
+echo "${bamboo_agent_ssh}" | base64 -d >> ~/.ssh/id_rsa
-    dvc pull
-
-    echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
-    echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
-    docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
-    docker build -f Dockerfile  -t nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} .
-    exit 0
-fi
-
-echo "gitTag=${newVersion}" > git.tag
 
 python3 -m venv build_venv
 source build_venv/bin/activate
 python3 -m pip install --upgrade pip
+pip install poetry
 
 pip install dvc
 pip install 'dvc[ssh]'
 echo "Pulling dvc data"
 dvc pull
 
-echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
+# update version in poetry to latest version in git if it is lower
-docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
+check_poetry_version () {
-docker build -f Dockerfile  -t nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} .
+    currentVersion=$(poetry version -s)
-echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
+    tagCount=$(git rev-list --tags --max-count=1 | wc -l)
-docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion}
+
+    if [[ $tagCount -gt 0 ]]
+    then
+        latestVersion=$(git rev-list --tags --max-count=1 | git describe --tags)
+        echo "latest version tag in git:    ${latestVersion}"
+        echo "current version in project:   ${currentVersion}"
+
+        if [[ "$currentVersion" < "$latestVersion" ]]
+        then
+            echo "project version is behind"
+            echo "setting latest git tag as current version"
+            poetry version "$latestVersion"
+            updateVersion=0
+        elif [[ "$currentVersion" == "$latestVersion" ]]
+        then
+            echo "project version matches"
+            echo "keeping the project version"
+            updateVersion=0
+        else
+            echo "project version is higher"
+            echo "keeping the project version, be aware that a version might have been skipped"
+            updateVersion=1
+        fi
+    else
+        updateVersion=1
+    fi
+
+    return $updateVersion
+}
+
+
+if [[ "$bamboo_planRepository_branchName" == "master" ]]
+then
+    if check_poetry_version
+    then
+        poetry version minor
+    fi
+    newVersion="$(poetry version -s)"
+    echo "new release on master with version:   $newVersion"
+elif [[ "$bamboo_planRepository_branchName" == release* ]]
+then
+    if check_poetry_version
+    then
+        poetry version patch
+    fi
+    newVersion="$(poetry version -s)"
+
+    echo "new release on $bamboo_planRepository_branchName with version $newVersion"
+elif [[ "${bamboo_version_tag}" != "dev" ]]
+then
+    newVersion="${bamboo_version_tag}"
+    echo "new special version build with:   $newVersion"
+else
+    newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
+    echo "gitTag=${newVersion}" > git.tag
+    echo "dev build with tag:   ${newVersion}"
+
+    echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@${HOST}/repository/python-combind/simple" >> pip.conf
+    echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin ${HOST}:${PORT}
+    docker build \
+        --build-arg SSH_KEY="$(cat ~/.ssh/id_rsa)" -f Dockerfile .
+    exit 0
+fi
+
+echo "gitTag=${newVersion}" > git.tag
+
+echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@${HOST}/repository/python-combind/simple" >> pip.conf
+echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin ${HOST}:${PORT}
+docker build \
+    -f Dockerfile \
+    -t ${HOST}:${PORT}/${project_name}/$SERVICE_NAME:${newVersion} \
+    --build-arg SSH_KEY="$(cat ~/.ssh/id_rsa)"
+docker push ${HOST}:${PORT}/${project_name}/$SERVICE_NAME:${newVersion}

incl/pdf2image

@@ -1 +0,0 @@
-Subproject commit f7292c30ad7c7ae5f07cee6925adda096301b60a

incl/pyinfra

@@ -1 +0,0 @@
-Subproject commit 64d6a8cec62eeddf26bd71a9aabc28b40dcec901

pyproject.toml

@@ -0,0 +1,51 @@
+[tool.poetry]
+name = "image-prediction"
+version = "1.21.1a0"
+description = "prediction service of the redai-image model"
+authors = ["Research Team <research@iqser.com>"]
+license = "all rights reserved"
+readme = "README.md"
+packages = [{include = "image_prediction", from = "src"}]
+
+[tool.poetry.dependencies]
+python = "~3.8"
+pip-licenses = "^3.5.4"
+pyinfra = {git = "ssh://git@git.iqser.com:2222/rr/pyinfra.git", branch = "master"}
+pdf2img = {git = "ssh://git@git.iqser.com:2222/rr/pdf2image.git", branch = "master"}
+tensorflow = "^2.11.0"
+pandas = "^1.5.3"
+frozendict = "^2.3.4"
+pdfnetpython3 = "^9.4.2"
+fpdf = "^1.7.2"
+funcy = "^1.17"
+requests = "^2.28.2"
+urllib3 = "^1.26.14"
+
+
+[tool.poetry.group.dev.dependencies]
+pylint = "^2.12.1"
+black = "^21.11b1"
+pytest = "^6.2.5"
+pytest-mock = "^3.6.1"
+ipykernel = "^6.6.0"
+pre-commit = "^2.16.0"
+isort = "^5.10.1"
+Sphinx = "^4.4.0"
+recommonmark = "^0.7.1"
+readthedocs-sphinx-ext = "^2.1.4"
+rinohtype = "^0.5.3"
+rst2pdf = "^0.99"
+pydata-sphinx-theme = "^0.8.0"
+docxbuilder = "^1.2.0"
+
+[tool.pytest.ini_options]
+norecursedirs = "incl"
+filterwarnings = ["ignore:.*:DeprecationWarning", ]
+ testpaths = [
+    "tests",
+]
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

pytest.ini

@@ -1,5 +0,0 @@
-[pytest]
-norecursedirs = incl
-filterwarnings =
-    ignore:.*:DeprecationWarning
-    ignore:.*:DeprecationWarning

requirements.txt

@@ -1,25 +0,0 @@
-Flask==2.1.1
-requests==2.27.1
-iteration-utilities==0.11.0
-dvc==2.10.0
-dvc[ssh]
-waitress==2.1.1
-envyaml==1.10.211231
-dependency-check==0.6.*
-mlflow==1.24.0
-numpy==1.22.3
-tqdm==4.64.0
-pandas==1.4.2
-tensorflow==2.8.0
-PyYAML==6.0
-pytest~=7.1.0
-funcy==1.17
-PyMuPDF==1.19.6
-fpdf==1.7.2
-coverage==6.3.2
-Pillow==9.1.0
-PDFNetPython3==9.1.0
-pdf2image==1.16.0
-frozendict==2.3.0
-protobuf<=3.20.*
-prometheus-client==0.13.1

scripts/pyinfra_mock.py

@@ -13,7 +13,6 @@ def parse_args():
 
 
 def main(args):
-
     response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb"))
     response.raise_for_status()
     predictions = response.json()

setup.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-
-from distutils.core import setup
-
-setup(
-    name="image_prediction",
-    version="0.1.0",
-    description="",
-    author="",
-    author_email="",
-    url="",
-    packages=["image_prediction"],
-)

src/image_prediction/__init__.py

@@ -0,0 +1,2 @@
+from . import *
+from image_prediction import *

src/image_prediction/locations.py

@@ -14,4 +14,4 @@ DATA_DIR = PACKAGE_ROOT_DIR / "data"
 
 MLRUNS_DIR = str(DATA_DIR / "mlruns")
 
-TEST_DATA_DIR = PACKAGE_ROOT_DIR / "test" / "data"
+TEST_DATA_DIR = "../" / PACKAGE_ROOT_DIR / "tests" / "data"

src/image_prediction/pipeline.py

@@ -6,7 +6,6 @@ from typing import Iterable
 from funcy import rcompose, first, compose, second, chunks, identity, rpartial
 from tqdm import tqdm
 
-from image_prediction.config import CONFIG
 from image_prediction.default_objects import (
     get_formatter,
     get_mlflow_model_loader,
@@ -14,19 +13,19 @@ from image_prediction.default_objects import (
     get_encoder,
     get_dispatched_extract,
 )
-from image_prediction.locations import MLRUNS_DIR
 from image_prediction.utils.generic import lift, starlift
 
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 
 
+def load_model(mlruns_dir, config):
+    model_loader = get_mlflow_model_loader(mlruns_dir)
+    model_identifier = config.service.mlflow_run_id
+    return get_image_classifier(model_loader, model_identifier)
+
+
 def load_pipeline(**kwargs):
-    model_loader = get_mlflow_model_loader(MLRUNS_DIR)
+    return Pipeline(**kwargs)
-    model_identifier = CONFIG.service.mlflow_run_id
-
-    pipeline = Pipeline(model_loader, model_identifier, **kwargs)
-
-    return pipeline
 
 
 def parallel(*fs):
@@ -38,16 +37,15 @@ def star(f):
 
 
 class Pipeline:
-    def __init__(self, model_loader, model_identifier, batch_size=16, verbose=True, **kwargs):
+    def __init__(self, model, batch_size=16, verbose=True, **kwargs):
         self.verbose = verbose
 
         extract = get_dispatched_extract(**kwargs)
-        classifier = get_image_classifier(model_loader, model_identifier)
         reformat = get_formatter()
         represent = get_encoder()
 
         split = compose(star(parallel(*map(lift, (first, first, second)))), rpartial(tee, 3))
-        classify = compose(chain.from_iterable, lift(classifier), partial(chunks, batch_size))
+        classify = compose(chain.from_iterable, lift(model), partial(chunks, batch_size))
         pairwise_apply = compose(star, parallel)
         join = compose(starlift(lambda prd, rpr, mdt: {"classification": prd, **mdt, "representation": rpr}), star(zip))
 

src/image_prediction/utils/pdf_annotation.py

@@ -55,9 +55,7 @@ def annotate_image(doc, image_info):
 
 
 def init():
-    PDFNet.Initialize(
+    PDFNet.Initialize("demo:1650351709282:7bd235e003000000004ec28a6743e1163a085e2115de2536ab6e2cfe5a")
-        "Knecon AG(en.knecon.swiss):OEM:DDA-R::WL+:AMS(20211029):BECC974307DAB4F34B513BC9B2531B24496F6FCB83CD8AC574358A959730B622FABEF5C7"
-    )
 
 
 def draw_metadata_box(pdf_path, metadata, store_path):