RES-731: add queues per tenant

.gitlab-ci.yml

@@ -12,15 +12,6 @@ variables:
   NEXUS_PROJECT_DIR: red # subfolder in Nexus docker-gin where your container will be stored
   IMAGENAME: $CI_PROJECT_NAME # if the project URL is gitlab.example.com/group-name/project-1, CI_PROJECT_NAME is project-1
 
-stages:
-  - data
-  - setup
-  - unit-tests
-  - versioning
-  - build
-  - integration-tests
-  - release
-
 pages:
   only:
     - master # KEEP THIS, necessary because `master` branch and not `main` branch

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ default_language_version:
   python: python3.10
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -34,7 +34,7 @@ repos:
           - --profile black
 
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
       - id: black
         # exclude: ^(docs/|notebooks/|data/|src/secrets/)

config/pyinfra.toml

@@ -1,9 +1,18 @@
+
+
+[dynamic_tenant_queues]
+enabled = true
+
 [metrics.prometheus]
 enabled = true
 prefix = "redactmanager_cv_analysis_service"
 
-[tracing.opentelemetry]
+[tracing]
 enabled = true
+# possible values "opentelemetry" | "azure_monitor" (Excpects APPLICATIONINSIGHTS_CONNECTION_STRING environment variable.)
+type = "azure_monitor"
+
+[tracing.opentelemetry]
 endpoint = "http://otel-collector-opentelemetry-collector.otel-collector:4318/v1/traces"
 service_name = "redactmanager_cv_analyisis_service"
 exporter = "otlp"
@@ -25,6 +34,15 @@ input_queue = "request_queue"
 output_queue = "response_queue"
 dead_letter_queue = "dead_letter_queue"
 
+tenant_event_queue_suffix = "_tenant_event_queue"
+tenant_event_dlq_suffix = "_tenant_events_dlq"
+tenant_exchange_name = "tenants-exchange"
+queue_expiration_time = 300000  # 5 minutes in milliseconds
+service_request_queue_prefix = "cv_analysis_request_queue"
+service_request_exchange_name = "cv_analysis_request_exchange"
+service_response_exchange_name = "cv_analysis_response_exchange"
+service_dlq_name = "cv_analysis_dlq"
+
 [storage]
 backend = "s3"
 
@@ -42,3 +60,6 @@ connection_string = ""
 [storage.tenant_server]
 public_key = ""
 endpoint =  "http://tenant-user-management:8081/internal-api/tenants"
+
+[kubernetes]
+pod_name = "test_pod"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cv-analysis-service"
-version = "2.19.0"
+version = "2.20.1"
 description = ""
 authors = ["Isaac Riley <isaac.riley@knecon.com>"]
 readme = "README.md"
@@ -25,7 +25,7 @@ coverage = "^5.5"
 dependency-check = "^0.6.0"
 lorem-text = "^2.1"
 PyMuPDF = "^1.19.6"
-pyinfra = { version = "^2.2.0", source = "gitlab-research" }
+pyinfra = { version = "3.2.2", source = "gitlab-research" }
 kn-utils = { version = "0.2.7", source = "gitlab-research" }
 pdf2img = { version = "0.7.0", source = "gitlab-red" }
 dvc-azure = "^2.21.2"