.gitlab-ci.yml

@@ -12,6 +12,15 @@ variables:
   NEXUS_PROJECT_DIR: red # subfolder in Nexus docker-gin where your container will be stored
   IMAGENAME: $CI_PROJECT_NAME # if the project URL is gitlab.example.com/group-name/project-1, CI_PROJECT_NAME is project-1
 
+stages:
+  - data
+  - setup
+  - unit-tests
+  - versioning
+  - build
+  - integration-tests
+  - release
+
 pages:
   only:
     - master # KEEP THIS, necessary because `master` branch and not `main` branch

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
   python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -34,7 +34,7 @@ repos:
           - --profile black
 
   - repo: https://github.com/psf/black
-    rev: 24.10.0
+    rev: 24.3.0
     hooks:
       - id: black
         # exclude: ^(docs/|notebooks/|data/|src/secrets/)
@@ -42,7 +42,7 @@ repos:
           - --line-length=120
 
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v4.0.0
+    rev: v3.2.0
     hooks:
       - id: conventional-pre-commit
         pass_filenames: false

config/pyinfra.toml

@@ -1,22 +1,11 @@
-
-[asyncio]
-max_concurrent_tasks = 10
-
-[dynamic_tenant_queues]
-enabled = true
-
 [metrics.prometheus]
 enabled = true
 prefix = "redactmanager_cv_analysis_service"
 
-[tracing]
-enabled = true
-# possible values "opentelemetry" | "azure_monitor" (Excpects APPLICATIONINSIGHTS_CONNECTION_STRING environment variable.)
-type = "azure_monitor"
-
 [tracing.opentelemetry]
+enabled = true
 endpoint = "http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces"
-service_name = "redactmanager_cv_analysis_service"
+service_name = "redactmanager_cv_analyisis_service"
 exporter = "otlp"
 
 [webserver]
@@ -36,15 +25,6 @@ input_queue = "request_queue"
 output_queue = "response_queue"
 dead_letter_queue = "dead_letter_queue"
 
-tenant_event_queue_suffix = "_tenant_event_queue"
-tenant_event_dlq_suffix = "_tenant_events_dlq"
-tenant_exchange_name = "tenants-exchange"
-queue_expiration_time = 300000                                   # 5 minutes in milliseconds
-service_request_queue_prefix = "cv_analysis_request_queue"
-service_request_exchange_name = "cv_analysis_request_exchange"
-service_response_exchange_name = "cv_analysis_response_exchange"
-service_dlq_name = "cv_analysis_dlq"
-
 [storage]
 backend = "s3"
 
@@ -61,7 +41,4 @@ connection_string = ""
 
 [storage.tenant_server]
 public_key = ""
-endpoint = "http://tenant-user-management:8081/internal-api/tenants"
-
-[kubernetes]
-pod_name = "test_pod"
+endpoint =  "http://tenant-user-management:8081/internal-api/tenants"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cv-analysis-service"
-version = "2.30.0"
+version = "2.16.1"
 description = ""
 authors = ["Isaac Riley <isaac.riley@knecon.com>"]
 readme = "README.md"
@@ -25,8 +25,8 @@ coverage = "^5.5"
 dependency-check = "^0.6.0"
 lorem-text = "^2.1"
 PyMuPDF = "^1.19.6"
-pyinfra = { version = "3.4.2", source = "gitlab-research" }
-kn-utils = { version = ">=0.4.0", source = "gitlab-research" }
+pyinfra = { version = "^2.2.0", source = "gitlab-research" }
+kn-utils = { version = "0.2.7", source = "gitlab-research" }
 pdf2img = { version = "0.7.0", source = "gitlab-red" }
 dvc-azure = "^2.21.2"
 pymupdf = "^1.24.1"
@@ -76,7 +76,7 @@ priority = "explicit"
 
 [tool.pylint]
 max-line-length = 120
-docstring-min-length = 4
+docstring-min-length=4
 extension-pkg-whitelist = ["cv2"]
 extension-pkg-allow-list = ["cv2"]
 

renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

scripts/devenvsetup.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-python_version=$1
-gitlab_user=$2
-gitlab_personal_access_token=$3
-
-# cookiecutter https://gitlab.knecon.com/knecon/research/template-python-project.git --checkout master
-# latest_dir=$(ls -td -- */ | head -n 1)  # should be the dir cookiecutter just created
-
-# cd $latest_dir
-
-pyenv install $python_version
-pyenv local $python_version
-pyenv shell $python_version
-
-# install poetry globally (PREFERRED), only need to install it once
-# curl -sSL https://install.python-poetry.org | python3 -
-
-# remember to update poetry once in a while
-poetry self update
-
-# install poetry in current python environment, can lead to multiple instances of poetry being installed on one system (DISPREFERRED)
-# pip install --upgrade pip
-# pip install poetry
-
-poetry config virtualenvs.in-project true
-poetry config installer.max-workers 10
-poetry config repositories.gitlab-research https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi
-poetry config http-basic.gitlab-research ${gitlab_user} ${gitlab_personal_access_token}
-poetry config repositories.gitlab-red https://gitlab.knecon.com/api/v4/groups/12/-/packages/pypi
-poetry config http-basic.gitlab-red ${gitlab_user} ${gitlab_personal_access_token}
-poetry config repositories.gitlab-fforesight https://gitlab.knecon.com/api/v4/groups/269/-/packages/pypi
-poetry config http-basic.gitlab-fforesight ${gitlab_user} ${gitlab_personal_access_token}
-
-poetry env use $(pyenv which python)
-poetry install --with=dev
-poetry update
-
-source .venv/bin/activate
-
-pre-commit install
-pre-commit autoupdate

src/cv_analysis/utils/annotate.py

@@ -4,9 +4,7 @@ from pathlib import Path
 from typing import Union
 
 import fitz  # type: ignore
-from kn_utils.logging import logger
-
-from cv_analysis.utils.image_extraction import mirror_horizontal  # type: ignore
+from kn_utils.logging import logger  # type: ignore
 
 
 def annotate_pdf(
@@ -30,8 +28,6 @@ def annotate_page(page: fitz.Page, prediction):
         label, probability, uuid = itemgetter("label", "probability", "uuid")(box)
 
         x0, y0, x1, y1 = bbox
-
-        print(page.bound)
         page.draw_rect(fitz.Rect(x0, y0, x1, y1), color=(0, 0, 1), width=2)
         label_x, label_y = x0, y0 - 5
         page.insert_text(
@@ -43,14 +39,6 @@ def annotate_page(page: fitz.Page, prediction):
     for line in prediction.get("tableLines", []):
         start = itemgetter("x1", "y1")(line)
         end = itemgetter("x2", "y2")(line)
-
-        bbox = (*start, *end)
-        height = page.bound()[3]
-        bbox = mirror_horizontal(bbox, page_height=height)
-
-        start = tuple(bbox[:2])
-        end = tuple(bbox[2:])
-
         page.draw_line(start, end, color=(1, 0, 0.5), width=1)
     return page
 

src/cv_analysis/utils/image_extraction.py

@@ -39,7 +39,7 @@ def transform_image_coordinates_to_pdf_coordinates(
     return rect.x0, rect.y0, rect.x1, rect.y1
 
 
-def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> BBoxType:
+def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> tuple[float, float, float, float]:
     round3 = lambda x: tuple(map(lambda y: round(y, 3), x))
 
     pdf_h, pdf_w = page_info.height, page_info.width
@@ -54,23 +54,20 @@ def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> BBoxType:
     return round3((bbox[0] * ratio_w, bbox[1] * ratio_h, bbox[2] * ratio_w, bbox[3] * ratio_h))
 
 
-def mirror_horizontal(bbox: BBoxType, page_height: int | float) -> BBoxType:
-    x0, y0, x1, y1 = bbox
-    y0_new = page_height - y0
-    y1_new = page_height - y1
+def derotate_image(bbox: tuple[float, float, float, float], page_info: PageInfo) -> ...:
+    def mirror_horizontal(bbox, page_height):
+        x0, y0, x1, y1 = bbox
+        y0_new = page_height - y1
+        y1_new = page_height - y0
 
-    return x0, y0_new, x1, y1_new
+        return x0, y0_new, x1, y1_new
 
+    def mirror_vertical(bbox, page_width):
+        x0, y0, x1, y1 = bbox
+        x0_new = page_width - x1
+        x1_new = page_width - x0
 
-def mirror_vertical(bbox: BBoxType, page_width: int | float) -> BBoxType:
-    x0, y0, x1, y1 = bbox
-    x0_new = page_width - x1
-    x1_new = page_width - x0
-
-    return x0_new, y0, x1_new, y1
-
-
-def derotate_image(bbox: BBoxType, page_info: PageInfo) -> BBoxType:
+        return x0_new, y0, x1_new, y1
 
     logger.debug(f"{page_info.rotation=}")
     match page_info.rotation:
@@ -106,7 +103,7 @@ def transform_table_lines_by_page_info(bboxes: dict, offsets: tuple, page_info:
 
     unpack = itemgetter("x1", "y1", "x2", "y2")
     pack = lambda x: {"x1": x[0], "y1": x[1], "x2": x[2], "y2": x[3]}
-    convert = compose(pack, derotate, apply_offsets, transform, unpack)
+    convert = compose(pack, apply_offsets, derotate, transform, unpack)
 
     table_lines = bboxes.get("tableLines", [])
     bboxes["tableLines"] = list(map(convert, table_lines))