Merge branch 'bugfix/RED-10722' into 'master'

RED-10722: fix dead letter queue

Closes RED-10722

See merge request redactmanager/cv-analysis-service!32

.gitignore

@@ -48,4 +48,5 @@ __pycache__/
 !data/*
 !drivers
 
-# ignore files
+# unignore files
+!bom.*

.gitlab-ci.yml

@@ -1,32 +1,35 @@
+# CI for services, check gitlab repo for python package CI
 include:
   - project: "Gitlab/gitlab"
-    ref: 0.3.0
+    ref: main
-    file: "/ci-templates/research/dvc-versioning-build-release.gitlab-ci.yml"
+    file: "/ci-templates/research/versioning-build-test-release.gitlab-ci.yml"
-    # file: "/ci-templates/research/versioning-build+azure_model-test-release.gitlab-ci.yml"
+  - project: "Gitlab/gitlab"
+    ref: main
+    file: "/ci-templates/research/docs.gitlab-ci.yml"
 
+# set project variables here
 variables:
-  NEXUS_PROJECT_DIR: red
+  NEXUS_PROJECT_DIR: red # subfolder in Nexus docker-gin where your container will be stored
-  IMAGENAME: "${CI_PROJECT_NAME}"
+  IMAGENAME: $CI_PROJECT_NAME # if the project URL is gitlab.example.com/group-name/project-1, CI_PROJECT_NAME is project-1
 
-#################################
+pages:
-# temp. disable integration tests, b/c they don't cover the CV analysis case yet
+  only:
-trigger integration tests:
+    - master # KEEP THIS, necessary because `master` branch and not `main` branch
+
+###################
+# INTEGRATION TESTS
+trigger-integration-tests:
+  extends: .integration-tests
+  # ADD THE MODEL BUILD WHICH SHOULD TRIGGER THE INTEGRATION TESTS
+  # needs:
+  #   - job: docker-build::model_name
+  #     artifacts: true
   rules:
     - when: never
 
-release build:
+#########
-  stage: release
+# RELEASE
+release:
+  extends: .release
   needs:
-  - job: set custom version
+    - !reference [.needs-versioning, needs] # leave this line as is
-    artifacts: true
-    optional: true
-  - job: calculate patch version
-    artifacts: true
-    optional: true
-  - job: calculate minor version
-    artifacts: true
-    optional: true
-  - job: build docker nexus
-    artifacts: true
-#################################
-

.hooks/poetry_version_check.py

@@ -13,9 +13,7 @@ logger.add(sys.stdout, level="INFO")
 def bashcmd(cmds: list) -> str:
     try:
         logger.debug(f"running: {' '.join(cmds)}")
-        return subprocess.run(
+        return subprocess.run(cmds, check=True, capture_output=True, text=True).stdout.strip("\n")
-            cmds, check=True, capture_output=True, text=True
-        ).stdout.strip("\n")
     except:
         logger.warning(f"Error executing the following bash command: {' '.join(cmds)}.")
         raise

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
   python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -34,7 +34,7 @@ repos:
           - --profile black
 
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         # exclude: ^(docs/|notebooks/|data/|src/secrets/)
@@ -42,7 +42,7 @@ repos:
           - --line-length=120
 
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: v3.2.0
+    rev: v4.0.0
     hooks:
       - id: conventional-pre-commit
         pass_filenames: false

Dockerfile

@@ -1,11 +1,19 @@
-FROM python:3.10
+###############
+# BUILDER IMAGE
+FROM python:3.10-slim as builder
+
+ARG GITLAB_USER
+ARG GITLAB_ACCESS_TOKEN
 
-ARG USERNAME
-ARG TOKEN
 ARG PYPI_REGISTRY_RESEARCH=https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi
 ARG POETRY_SOURCE_REF_RESEARCH=gitlab-research
+
 ARG PYPI_REGISTRY_RED=https://gitlab.knecon.com/api/v4/groups/12/-/packages/pypi
 ARG POETRY_SOURCE_REF_RED=gitlab-red
+
+ARG PYPI_REGISTRY_FFORESIGHT=https://gitlab.knecon.com/api/v4/groups/269/-/packages/pypi
+ARG POETRY_SOURCE_REF_FFORESIGHT=gitlab-fforesight
+
 ARG VERSION=dev
 
 LABEL maintainer="Research <research@knecon.com>"
@@ -13,29 +21,58 @@ LABEL version="${VERSION}"
 
 WORKDIR /app
 
+###########
+# ENV SETUP
+ENV PYTHONDONTWRITEBYTECODE=true
 ENV PYTHONUNBUFFERED=true
 ENV POETRY_HOME=/opt/poetry
 ENV PATH="$POETRY_HOME/bin:$PATH"
 
-RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN apt-get update && \
+    apt-get install -y curl git bash build-essential libffi-dev libssl-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://install.python-poetry.org | python3 -
+RUN poetry --version
 
-COPY ./src ./src
-COPY ./config ./config
 COPY pyproject.toml poetry.lock ./
 
-
+RUN poetry config virtualenvs.create true && \
-RUN ls -hal
+    poetry config virtualenvs.in-project true && \
-RUN echo "${USERNAME} ${TOKEN} ${PYPI_REGISTRY_RED} ${POETRY_SOURCE_REF_RED} "
-
-RUN poetry config virtualenvs.create false && \
     poetry config installer.max-workers 10 && \
     poetry config repositories.${POETRY_SOURCE_REF_RESEARCH} ${PYPI_REGISTRY_RESEARCH} && \
-    poetry config http-basic.${POETRY_SOURCE_REF_RESEARCH} ${USERNAME} ${TOKEN} && \
+    poetry config http-basic.${POETRY_SOURCE_REF_RESEARCH} ${GITLAB_USER} ${GITLAB_ACCESS_TOKEN} && \
     poetry config repositories.${POETRY_SOURCE_REF_RED} ${PYPI_REGISTRY_RED} && \
-    poetry config http-basic.${POETRY_SOURCE_REF_RED} ${USERNAME} ${TOKEN} && \
+    poetry config http-basic.${POETRY_SOURCE_REF_RED} ${GITLAB_USER} ${GITLAB_ACCESS_TOKEN} && \
-    poetry install --without=test -vv --no-interaction
+    poetry config repositories.${POETRY_SOURCE_REF_FFORESIGHT} ${PYPI_REGISTRY_FFORESIGHT} && \
+    poetry config http-basic.${POETRY_SOURCE_REF_FFORESIGHT} ${GITLAB_USER} ${GITLAB_ACCESS_TOKEN} && \
+    poetry install --without=dev,docs,test -vv --no-interaction --no-root
 
+##################
+# COPY SOURCE CODE
+COPY ./config ./config
+COPY ./src ./src
+
+###############
+# WORKING IMAGE
+FROM python:3.10-slim
+
+# COPY BILL OF MATERIALS (BOM)
+COPY bom.json /bom.json
+
+# COPY SOURCE CODE FROM BUILDER IMAGE
+COPY --from=builder /app /app
+
+WORKDIR /app
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+############
+# NETWORKING
 EXPOSE 5000
 EXPOSE 8080
 
+################
+# LAUNCH COMMAND
 CMD [ "python", "src/serve.py"]

Makefile

@@ -4,7 +4,7 @@
 	docker docker-build-run docker-build docker-run \
 	docker-rm docker-rm-container docker-rm-image \
 	pre-commit get-licenses prep-commit \
-	docs sphinx_html sphinx_apidoc
+	docs sphinx_html sphinx_apidoc bom
 .DEFAULT_GOAL := run
 
 export DOCKER=docker
@@ -89,3 +89,6 @@ sphinx_html:
 
 sphinx_apidoc:
 	cp ./README.md ./docs/source/README.md && cp -r ./data ./docs/source/data/ && poetry run sphinx-apidoc ./src -o ./docs/source/modules --no-toc --module-first --follow-links --separate --force
+
+bom:
+	cyclonedx-py poetry -o bom.json

config/pyinfra.toml

@@ -1,11 +1,22 @@
+
+[asyncio]
+max_concurrent_tasks = 10
+
+[dynamic_tenant_queues]
+enabled = true
+
 [metrics.prometheus]
 enabled = true
 prefix = "redactmanager_cv_analysis_service"
 
-[tracing.opentelemetry]
+[tracing]
 enabled = true
+# possible values "opentelemetry" | "azure_monitor" (Excpects APPLICATIONINSIGHTS_CONNECTION_STRING environment variable.)
+type = "azure_monitor"
+
+[tracing.opentelemetry]
 endpoint = "http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces"
-service_name = "redactmanager_cv_analyisis_service"
+service_name = "redactmanager_cv_analysis_service"
 exporter = "otlp"
 
 [webserver]
@@ -25,6 +36,15 @@ input_queue = "request_queue"
 output_queue = "response_queue"
 dead_letter_queue = "dead_letter_queue"
 
+tenant_event_queue_suffix = "_tenant_event_queue"
+tenant_event_dlq_suffix = "_tenant_events_dlq"
+tenant_exchange_name = "tenants-exchange"
+queue_expiration_time = 300000                                   # 5 minutes in milliseconds
+service_request_queue_prefix = "cv_analysis_request_queue"
+service_request_exchange_name = "cv_analysis_request_exchange"
+service_response_exchange_name = "cv_analysis_response_exchange"
+service_dlq_name = "cv_analysis_dlq"
+
 [storage]
 backend = "s3"
 
@@ -42,3 +62,6 @@ connection_string = ""
 [storage.tenant_server]
 public_key = ""
 endpoint = "http://tenant-user-management:8081/internal-api/tenants"
+
+[kubernetes]
+pod_name = "test_pod"

flake.lock

@@ -20,17 +20,17 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1711703276,
+        "lastModified": 1715155958,
-        "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
+        "narHash": "sha256-I/V8oiPfK0KIQUc+3sAQLJJYa7L3edd9gdnKP2XvT7E=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
+        "rev": "240b1d794bbfca3522dec880a3aa300932bbfd98",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-unstable",
         "repo": "nixpkgs",
+        "rev": "240b1d794bbfca3522dec880a3aa300932bbfd98",
         "type": "github"
       }
     },

flake.nix

@@ -2,7 +2,7 @@
   description = "An flake to use a Python poetry project in an FHS environment when poetry2nix is uncooperative";
   inputs = {
     flake-utils.url = "github:numtide/flake-utils";
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/240b1d794bbfca3522dec880a3aa300932bbfd98";
   };
   outputs = {
     self,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cv-analysis-service"
-version = "2.6.1"
+version = "2.30.0"
 description = ""
 authors = ["Isaac Riley <isaac.riley@knecon.com>"]
 readme = "README.md"
@@ -25,8 +25,8 @@ coverage = "^5.5"
 dependency-check = "^0.6.0"
 lorem-text = "^2.1"
 PyMuPDF = "^1.19.6"
-pyinfra = { version = "^2.2.0", source = "gitlab-research" }
+pyinfra = { version = "3.4.2", source = "gitlab-research" }
-kn-utils = { version = "0.2.7", source = "gitlab-research" }
+kn-utils = { version = ">=0.4.0", source = "gitlab-research" }
 pdf2img = { version = "0.7.0", source = "gitlab-red" }
 dvc-azure = "^2.21.2"
 pymupdf = "^1.24.1"
@@ -43,6 +43,7 @@ mypy = "^1.10.0"
 pylint = "^3.1.0"
 pre-commit = "^3.7.0"
 semver = "^3.0.2"
+cyclonedx-bom = "^4.4.3"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^7.2.6"

scripts/devenvsetup.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+python_version=$1
+gitlab_user=$2
+gitlab_personal_access_token=$3
+
+# cookiecutter https://gitlab.knecon.com/knecon/research/template-python-project.git --checkout master
+# latest_dir=$(ls -td -- */ | head -n 1)  # should be the dir cookiecutter just created
+
+# cd $latest_dir
+
+pyenv install $python_version
+pyenv local $python_version
+pyenv shell $python_version
+
+# install poetry globally (PREFERRED), only need to install it once
+# curl -sSL https://install.python-poetry.org | python3 -
+
+# remember to update poetry once in a while
+poetry self update
+
+# install poetry in current python environment, can lead to multiple instances of poetry being installed on one system (DISPREFERRED)
+# pip install --upgrade pip
+# pip install poetry
+
+poetry config virtualenvs.in-project true
+poetry config installer.max-workers 10
+poetry config repositories.gitlab-research https://gitlab.knecon.com/api/v4/groups/19/-/packages/pypi
+poetry config http-basic.gitlab-research ${gitlab_user} ${gitlab_personal_access_token}
+poetry config repositories.gitlab-red https://gitlab.knecon.com/api/v4/groups/12/-/packages/pypi
+poetry config http-basic.gitlab-red ${gitlab_user} ${gitlab_personal_access_token}
+poetry config repositories.gitlab-fforesight https://gitlab.knecon.com/api/v4/groups/269/-/packages/pypi
+poetry config http-basic.gitlab-fforesight ${gitlab_user} ${gitlab_personal_access_token}
+
+poetry env use $(pyenv which python)
+poetry install --with=dev
+poetry update
+
+source .venv/bin/activate
+
+pre-commit install
+pre-commit autoupdate

scripts/parse_pdf.py

@@ -26,4 +26,5 @@ best_result = list(pipe(data={"pdf": pdf_bytes, "vlp_output": vlp_output}))
 
 # print(best_result)
 
+if __name__ == "__main__":
     annotate_pdf(pdf_bytes, best_result, output_path=args.output)

src/cv_analysis/table_inference.py

@@ -135,9 +135,11 @@ FILTERS: dict[str, dict[int, np.ndarray | None]] = {
 
 
 def filter_fp_col_lines(line_list: list[int], filt_sums: Array) -> list[int]:
-    if not line_list:
+    if not list(line_list):
         return []
     centers = list(np.where((filt_sums[1:-1] < filt_sums[:-2]) * (filt_sums[1:-1] < filt_sums[2:]))[0] + 1)
+    if not centers:
+        return []
 
     if line_list[0] > centers[0]:
         centers = centers[1:] + [len(filt_sums) - 1]

src/cv_analysis/utils/annotate.py

@@ -4,7 +4,9 @@ from pathlib import Path
 from typing import Union
 
 import fitz  # type: ignore
-from kn_utils.logging import logger  # type: ignore
+from kn_utils.logging import logger
+
+from cv_analysis.utils.image_extraction import mirror_horizontal  # type: ignore
 
 
 def annotate_pdf(
@@ -27,8 +29,9 @@ def annotate_page(page: fitz.Page, prediction):
         bbox = itemgetter("x1", "y1", "x2", "y2")(box["box"])
         label, probability, uuid = itemgetter("label", "probability", "uuid")(box)
 
-        bbox = mirror_on_x_axis(bbox, page.bound().height)
         x0, y0, x1, y1 = bbox
+
+        print(page.bound)
         page.draw_rect(fitz.Rect(x0, y0, x1, y1), color=(0, 0, 1), width=2)
         label_x, label_y = x0, y0 - 5
         page.insert_text(
@@ -40,18 +43,18 @@ def annotate_page(page: fitz.Page, prediction):
     for line in prediction.get("tableLines", []):
         start = itemgetter("x1", "y1")(line)
         end = itemgetter("x2", "y2")(line)
+
+        bbox = (*start, *end)
+        height = page.bound()[3]
+        bbox = mirror_horizontal(bbox, page_height=height)
+
+        start = tuple(bbox[:2])
+        end = tuple(bbox[2:])
+
         page.draw_line(start, end, color=(1, 0, 0.5), width=1)
     return page
 
 
-def mirror_on_x_axis(bbox, page_height):
-    x0, y0, x1, y1 = bbox
-    y0_new = page_height - y1
-    y1_new = page_height - y0
-
-    return x0, y0_new, x1, y1_new
-
-
 @singledispatch
 def provide_byte_stream(pdf: Union[bytes, Path, str]) -> None:
     pass

src/cv_analysis/utils/display.py

@@ -3,10 +3,10 @@ import os
 import cv2
 from matplotlib import pyplot as plt
 
-if os.environ["USER"] == "isaac":
+# if os.environ.get("USER") == "isaac":
-    import matplotlib
+#     import matplotlib
 
-    matplotlib.use("module://matplotlib-backend-wezterm")
+#     matplotlib.use("module://matplotlib-backend-wezterm")
 
 
 def show_image_cv2(image, maxdim=700):

src/cv_analysis/utils/image_extraction.py

@@ -2,7 +2,7 @@ import json
 from dataclasses import dataclass
 from functools import partial
 from operator import itemgetter
-from typing import SupportsIndex, Tuple
+from typing import Literal, SupportsIndex, Tuple
 
 import fitz  # type: ignore
 import numpy as np
@@ -39,7 +39,7 @@ def transform_image_coordinates_to_pdf_coordinates(
     return rect.x0, rect.y0, rect.x1, rect.y1
 
 
-def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> tuple[float, float, float, float]:
+def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> BBoxType:
     round3 = lambda x: tuple(map(lambda y: round(y, 3), x))
 
     pdf_h, pdf_w = page_info.height, page_info.width
@@ -54,6 +54,41 @@ def rescale_to_pdf(bbox: BBoxType, page_info: PageInfo) -> tuple[float, float, f
     return round3((bbox[0] * ratio_w, bbox[1] * ratio_h, bbox[2] * ratio_w, bbox[3] * ratio_h))
 
 
+def mirror_horizontal(bbox: BBoxType, page_height: int | float) -> BBoxType:
+    x0, y0, x1, y1 = bbox
+    y0_new = page_height - y0
+    y1_new = page_height - y1
+
+    return x0, y0_new, x1, y1_new
+
+
+def mirror_vertical(bbox: BBoxType, page_width: int | float) -> BBoxType:
+    x0, y0, x1, y1 = bbox
+    x0_new = page_width - x1
+    x1_new = page_width - x0
+
+    return x0_new, y0, x1_new, y1
+
+
+def derotate_image(bbox: BBoxType, page_info: PageInfo) -> BBoxType:
+
+    logger.debug(f"{page_info.rotation=}")
+    match page_info.rotation:
+        case 0:
+            bbox = mirror_horizontal(bbox, page_info.height)
+        case 90:
+            pass
+        case 180:
+            bbox = mirror_vertical(bbox, page_info.height)
+        case 270:
+            bbox = mirror_vertical(mirror_horizontal(bbox, page_info.height), page_info.height)
+        case _:
+            logger.warning(f"Unknown rotation: {page_info.rotation}")
+            pass
+
+    return bbox
+
+
 def transform_table_lines_by_page_info(bboxes: dict, offsets: tuple, page_info: PageInfo) -> dict:
 
     transform = partial(rescale_to_pdf, page_info=page_info)
@@ -67,9 +102,11 @@ def transform_table_lines_by_page_info(bboxes: dict, offsets: tuple, page_info:
 
         return (x1 + offset_x, y1 + offset_y, x2 + offset_x, y2 + offset_y)
 
+    derotate = partial(derotate_image, page_info=page_info)
+
     unpack = itemgetter("x1", "y1", "x2", "y2")
     pack = lambda x: {"x1": x[0], "y1": x[1], "x2": x[2], "y2": x[3]}
-    convert = compose(pack, apply_offsets, transform, unpack)
+    convert = compose(pack, derotate, apply_offsets, transform, unpack)
 
     table_lines = bboxes.get("tableLines", [])
     bboxes["tableLines"] = list(map(convert, table_lines))