chore: repo housekeeping: adapt pre-commit and versioning script

.coveragerc

@@ -10,7 +10,7 @@ omit =
     */build_venv/*
 	*/incl/*
 source =
-    cv_analysis 
+    cv_analysis
 relative_files = True
 data_file = .coverage
 
@@ -46,4 +46,4 @@ ignore_errors = True
 directory = reports
 
 [xml]
-output = reports/coverage.xml
+output = reports/coverage.xml

.dockerignore

@@ -97,4 +97,4 @@ target/
 *.swp
 */*.swp
 */*/*.swp
-*/*/*/*.swp
+*/*/*/*.swp

.dvc/config

@@ -8,6 +8,3 @@
         connection_string = "DefaultEndpointsProtocol=https;AccountName=cvsacricket;AccountKey=KOuTAQ6Mp00ePTT5ObYmgaHlxwS1qukY4QU4Kuk7gy/vldneA+ZiKjaOpEFtqKA6Mtym2gQz8THy+ASts/Y1Bw==;EndpointSuffix=core.windows.net"
 ['remote "local"']
     url = ../dvc_local_remote
-
-        
-

.hooks/poetry_version_check.py

@@ -0,0 +1,63 @@
+import subprocess
+import sys
+from pathlib import Path
+
+import semver
+from loguru import logger
+from semver.version import Version
+
+logger.remove()
+logger.add(sys.stdout, level="INFO")
+
+
+def bashcmd(cmds: list) -> str:
+    try:
+        logger.debug(f"running: {' '.join(cmds)}")
+        return subprocess.run(
+            cmds, check=True, capture_output=True, text=True
+        ).stdout.strip("\n")
+    except:
+        logger.warning(f"Error executing the following bash command: {' '.join(cmds)}.")
+        raise
+
+
+def get_highest_existing_git_version_tag() -> str:
+    """Get highest versions from git tags depending on bump level"""
+    try:
+        git_tags = bashcmd(["git", "tag", "-l"]).split()
+        semver_compat_tags = list(filter(Version.is_valid, git_tags))
+        highest_git_version_tag = max(semver_compat_tags, key=semver.version.Version.parse)
+        logger.info(f"Highest git version tag: {highest_git_version_tag}")
+        return highest_git_version_tag
+    except:
+        logger.warning("Error getting git version tags")
+        raise
+
+
+def auto_bump_version() -> bool:
+    active = Path(".autoversion").is_file()
+    logger.debug(f"Automated version bump is set to '{active}'")
+    return active
+
+
+def main() -> None:
+    poetry_project_version = bashcmd(["poetry", "version", "-s"])
+
+    logger.info(f"Poetry project version: {poetry_project_version}")
+
+    highest_git_version_tag = get_highest_existing_git_version_tag()
+
+    comparison_result = semver.compare(poetry_project_version, highest_git_version_tag)
+
+    if comparison_result in (-1, 0):
+        logger.warning("Poetry version must be greater than git tag version.")
+        if auto_bump_version():
+            logger.info(bashcmd(["poetry", "version", highest_git_version_tag]))
+            sys.exit(0)
+        sys.exit(1)
+    else:
+        logger.info(f"All good: {poetry_project_version} > {highest_git_version_tag}")
+
+
+if __name__ == "__main__":
+    main()

.pre-commit-config.yaml

@@ -0,0 +1,71 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+exclude: ^(docs/|notebooks/|data/|src/configs/|tests/|.hooks/|bom.json)
+default_language_version:
+  python: python3.10
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=10000']
+      - id: check-case-conflict
+      - id: mixed-line-ending
+
+  # - repo: https://github.com/pre-commit/mirrors-pylint
+  #   rev: v3.0.0a5
+  #   hooks:
+  #     - id: pylint
+  #       args:
+  #         - --disable=C0111,R0903,E0401
+  #         - --max-line-length=120
+
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.10.1
+    hooks:
+      - id: isort
+        args:
+          - --profile black
+
+  - repo: https://github.com/psf/black
+    rev: 24.3.0
+    hooks:
+      - id: black
+        # exclude: ^(docs/|notebooks/|data/|src/secrets/)
+        args:
+          - --line-length=120
+
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v3.2.0
+    hooks:
+      - id: conventional-pre-commit
+        pass_filenames: false
+        stages: [commit-msg]
+        # args: [] # optional: list of Conventional Commits types to allow e.g. [feat, fix, ci, chore, test]
+
+  - repo: local
+    hooks:
+      - id: version-checker
+        name: version-checker
+        entry: python .hooks/poetry_version_check.py
+        language: python
+        always_run: true
+        additional_dependencies:
+          - "semver"
+          - "loguru"
+
+  # - repo: local
+  #   hooks:
+  #     - id: docker-build-test
+  #       name: testing docker build
+  #       entry: ./scripts/ops/docker-compose-build-run.sh
+  #       language: script
+  #       # always_run: true
+  #       pass_filenames: false
+  #       args: []
+  #       stages: [pre-commit]

config/pyinfra.toml

@@ -41,4 +41,4 @@ connection_string = ""
 
 [storage.tenant_server]
 public_key = ""
-endpoint =  "http://tenant-user-management:8081/internal-api/tenants"
+endpoint =  "http://tenant-user-management:8081/internal-api/tenants"

docker-compose.yaml

@@ -28,4 +28,4 @@ services:
     volumes:
       - /opt/bitnami/rabbitmq/.rabbitmq/:/data/bitnami
 volumes:
-  mdata:
+  mdata:

poetry.lock

@@ -677,6 +677,17 @@ files = [
 [package.dependencies]
 pycparser = "*"
 
+[[package]]
+name = "cfgv"
+version = "3.4.0"
+description = "Validate configuration and produce human readable error messages."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
+    {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.3.2"
@@ -1142,6 +1153,17 @@ files = [
     {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"},
 ]
 
+[[package]]
+name = "distlib"
+version = "0.3.8"
+description = "Distribution utilities"
+optional = false
+python-versions = "*"
+files = [
+    {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"},
+    {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"},
+]
+
 [[package]]
 name = "distro"
 version = "1.9.0"
@@ -1973,6 +1995,20 @@ antlr4-python3-runtime = "==4.9.*"
 omegaconf = ">=2.2,<2.4"
 packaging = "*"
 
+[[package]]
+name = "identify"
+version = "2.5.36"
+description = "File identification library for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"},
+    {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"},
+]
+
+[package.extras]
+license = ["ukkonen"]
+
 [[package]]
 name = "idna"
 version = "3.7"
@@ -2855,6 +2891,20 @@ doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-t
 extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"]
 test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"]
 
+[[package]]
+name = "nodeenv"
+version = "1.8.0"
+description = "Node.js virtual environment builder"
+optional = false
+python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
+files = [
+    {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"},
+    {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"},
+]
+
+[package.dependencies]
+setuptools = "*"
+
 [[package]]
 name = "numpy"
 version = "1.26.4"
@@ -3517,6 +3567,24 @@ docs = ["sphinx (>=1.7.1)"]
 redis = ["redis"]
 tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"]
 
+[[package]]
+name = "pre-commit"
+version = "3.7.0"
+description = "A framework for managing and maintaining multi-language pre-commit hooks."
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"},
+    {file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"},
+]
+
+[package.dependencies]
+cfgv = ">=2.0.0"
+identify = ">=1.0.0"
+nodeenv = ">=0.11.1"
+pyyaml = ">=5.1"
+virtualenv = ">=20.10.0"
+
 [[package]]
 name = "prometheus-client"
 version = "0.18.0"
@@ -4361,6 +4429,17 @@ shortuuid = ">=0.5.0"
 dev = ["mock (==5.1.0)", "mypy (==0.971)", "paramiko (==3.3.1)", "pylint (==2.15.0)", "pytest (==7.2.0)", "pytest-asyncio (==0.18.3)", "pytest-cov (==3.0.0)", "pytest-docker (==0.12.0)", "pytest-mock (==3.8.2)", "pytest-sugar (==0.9.5)", "pytest-test-utils (==0.0.8)", "types-certifi (==2021.10.8.3)", "types-mock (==5.1.0.2)", "types-paramiko (==3.3.0.0)"]
 tests = ["mock (==5.1.0)", "mypy (==0.971)", "paramiko (==3.3.1)", "pylint (==2.15.0)", "pytest (==7.2.0)", "pytest-asyncio (==0.18.3)", "pytest-cov (==3.0.0)", "pytest-docker (==0.12.0)", "pytest-mock (==3.8.2)", "pytest-sugar (==0.9.5)", "pytest-test-utils (==0.0.8)", "types-certifi (==2021.10.8.3)", "types-mock (==5.1.0.2)", "types-paramiko (==3.3.0.0)"]
 
+[[package]]
+name = "semver"
+version = "3.0.2"
+description = "Python helper for Semantic Versioning (https://semver.org)"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"},
+    {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"},
+]
+
 [[package]]
 name = "setuptools"
 version = "69.5.1"
@@ -4669,6 +4748,26 @@ files = [
     {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
 ]
 
+[[package]]
+name = "virtualenv"
+version = "20.26.0"
+description = "Virtual Python Environment builder"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "virtualenv-20.26.0-py3-none-any.whl", hash = "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3"},
+    {file = "virtualenv-20.26.0.tar.gz", hash = "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210"},
+]
+
+[package.dependencies]
+distlib = ">=0.3.7,<1"
+filelock = ">=3.12.2,<4"
+platformdirs = ">=3.9.1,<5"
+
+[package.extras]
+docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
+test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"]
+
 [[package]]
 name = "voluptuous"
 version = "0.14.2"
@@ -4954,4 +5053,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.10,<3.11"
-content-hash = "06b9635bd0acdb0cd78ee9ab0e6a8c75ef91648c36bbc1f78ac44a5c0671990b"
+content-hash = "971cfd0d8034dcf0cefd68d8fff173a36a83fe42625b5eca31d4b5275fbc8e1f"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "cv-analysis-service"
-version = "2.1.0"
+version = "2.5.1"
 description = ""
 authors = []
 readme = "README.md"
@@ -41,6 +41,8 @@ pylint = "^3.1"
 ipython = "^8.21.0"
 mypy = "^1.10.0"
 pylint = "^3.1.0"
+pre-commit = "^3.7.0"
+semver = "^3.0.2"
 
 [tool.pytest.ini_options]
 testpaths = ["test"]
@@ -66,6 +68,12 @@ docstring-min-length=4
 extension-pkg-whitelist = ["cv2"]
 extension-pkg-allow-list = ["cv2"]
 
+[tool.black]
+line-length = 120
+
+[tool.isort]
+profile = "black"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

scripts/annotate.py

@@ -45,7 +45,7 @@ if __name__ == "__main__":
     elif args.type == "layout":
         from cv_analysis.layout_parsing import parse_layout as analyze
     elif args.type == "figure":
-        from cv_analysis.figure_detection.figure_detection import \
+        from cv_analysis.figure_detection.figure_detection import detect_figures
-            detect_figures
+
         analyze = detect_figures
     annotate_page(page, analyze, draw, name=name, show=args.show)

scripts/annotate_pdf.py

@@ -56,7 +56,6 @@ if __name__ == "__main__":
     t2 = timeit.default_timer()
     save_as_pdf(annotated_pages, args.output_folder, Path(args.pdf_path).stem, args.type)
     t3 = timeit.default_timer()
-    print("[s] opening file and convert pdf pages to images: ", t1-t0)
+    print("[s] opening file and convert pdf pages to images: ", t1 - t0)
-    print("[s] analyse and annotate images: ", t2-t1)
+    print("[s] analyse and annotate images: ", t2 - t1)
-    print("[s] save images as pdf: ", t3-t2)
+    print("[s] save images as pdf: ", t3 - t2)
-

scripts/docker_build_run.sh

@@ -1,4 +1,4 @@
-docker build -t cv-analysis-service:$(poetry version -s)-dev \                                                               
+docker build -t cv-analysis-service:$(poetry version -s)-dev \
     -f Dockerfile \
     --build-arg USERNAME=$GITLAB_USER \
     --build-arg TOKEN=$GITLAB_ACCESS_TOKEN \

scripts/explore_aio_detection.py

@@ -94,7 +94,7 @@ def classify_node_recursively(image, node):
     if maybe_texts:
         children = lmap(lambda b: make_child(b, node, ["text"]), maybe_texts)
         node.children = children
-        return node  #FIGURES
+        return node  # FIGURES
 
     return node
 

src/cv_analysis/figure_detection/figure_detection.py

@@ -5,7 +5,11 @@ import numpy as np
 
 from cv_analysis.figure_detection.figures import detect_large_coherent_structures
 from cv_analysis.figure_detection.text import remove_primary_text_regions
-from cv_analysis.utils.filters import has_acceptable_format, is_large_enough, is_not_too_large
+from cv_analysis.utils.filters import (
+    has_acceptable_format,
+    is_large_enough,
+    is_not_too_large,
+)
 from cv_analysis.utils.postprocessing import remove_included
 from cv_analysis.utils.structures import Rectangle
 

src/cv_analysis/layout_parsing.py

@@ -6,7 +6,11 @@ import cv2
 import numpy as np
 
 from cv_analysis.utils.connect_rects import connect_related_rects2
-from cv_analysis.utils.postprocessing import has_no_parent, remove_included, remove_overlapping
+from cv_analysis.utils.postprocessing import (
+    has_no_parent,
+    remove_included,
+    remove_overlapping,
+)
 from cv_analysis.utils.structures import Rectangle
 from cv_analysis.utils.visual_logging import vizlogger
 

src/cv_analysis/server/pipeline.py

@@ -10,7 +10,10 @@ from pdf2img.default_objects.rectangle import RectanglePlus
 from cv_analysis.figure_detection.figure_detection import detect_figures
 from cv_analysis.table_inference import infer_lines
 from cv_analysis.table_parsing import parse_lines, parse_tables
-from cv_analysis.utils.image_extraction import extract_images_from_pdf, transform_table_lines_by_page_info
+from cv_analysis.utils.image_extraction import (
+    extract_images_from_pdf,
+    transform_table_lines_by_page_info,
+)
 from cv_analysis.utils.structures import Rectangle
 
 
@@ -49,8 +52,7 @@ def make_image_analysis_pipeline(
         # rel_bboxes = map()
         img_results = lmap(analysis_fn, images)
 
-        def make_offsets():
+        def make_offsets(): ...
-            ...
 
         offsets = map(itemgetter("x1", "y2"), map(itemgetter("bbox"), info))
         # print("before", img_results)

src/cv_analysis/table_parsing.py

@@ -3,7 +3,9 @@ from cv2 import cv2
 from funcy import lfilter, lmap  # type: ignore
 
 from cv_analysis.layout_parsing import parse_layout
-from cv_analysis.utils.postprocessing import remove_isolated  # xywh_to_vecs, xywh_to_vec_rect, adjacent1d
+from cv_analysis.utils.postprocessing import (
+    remove_isolated,  # xywh_to_vecs, xywh_to_vec_rect, adjacent1d
+)
 from cv_analysis.utils.structures import Rectangle
 from cv_analysis.utils.visual_logging import vizlogger
 

src/cv_analysis/utils/banner.py

@@ -1,13 +1,13 @@
 def make_art():
     art = r"""
-       __           
+       __
-   _  |@@|          
+   _  |@@|
   / \ \--/ __                                            .__               .__
   ) O|----|  |   __ ___  __         _____    ____ _____  |  | ___.__. _____|__| ______
  / / \ }{ /\ )_ / _\\  \/ /  ______ \__  \  /    \\__  \ |  | |  |  |/  ___/  |/  ___/
  )/  /\__/\ \__O (__ \   /  /_____/  / __ \|   |  \/ __ \|  |_\___  |\___ \|  |\___ \
-|/  (--/\--)    \__/  \_/           (______/___|__(______/____/\____/_____/|__/_____/ 
+|/  (--/\--)    \__/  \_/           (______/___|__(______/____/\____/_____/|__/_____/
-/   _)(  )(_                                                                          
+/   _)(  )(_
-   `---''---`       
+   `---''---`
 """
     return art

test/unit_tests/figure_detection/text_test.py

@@ -4,8 +4,10 @@ import cv2
 import numpy as np
 import pytest
 
-from cv_analysis.figure_detection.text import (apply_threshold_to_image,
+from cv_analysis.figure_detection.text import (
-                                               remove_primary_text_regions)
+    apply_threshold_to_image,
+    remove_primary_text_regions,
+)
 
 
 @pytest.mark.parametrize("error_tolerance", [0.07])

test/unit_tests/server_pipeline_test.py

@@ -2,9 +2,11 @@ import fitz
 import numpy as np
 import pytest
 
-from cv_analysis.server.pipeline import (figure_detection_formatter,
+from cv_analysis.server.pipeline import (
-                                         make_analysis_pipeline,
+    figure_detection_formatter,
-                                         table_parsing_formatter)
+    make_analysis_pipeline,
+    table_parsing_formatter,
+)
 from cv_analysis.utils.structures import Rectangle
 
 
@@ -78,8 +80,6 @@ def formatter(operation):
 
 @pytest.mark.parametrize("operation", ["figure"])
 def test_analysis_pipeline(empty_pdf, formatter, expected_formatted_analysis_result):
-    analysis_pipeline = make_analysis_pipeline(
+    analysis_pipeline = make_analysis_pipeline(analysis_fn_mock, formatter, dpi=200, skip_pages_without_images=False)
-        analysis_fn_mock, formatter, dpi=200, skip_pages_without_images=False
-    )
     results = list(analysis_pipeline(empty_pdf))
     assert list(results) == expected_formatted_analysis_result

test/unit_tests/table_inference_test.py

@@ -11,7 +11,11 @@ def test_table_inference_smoke():
             {
                 "page_idx": 1,
                 "boxes": [
-                    {"uuid": "marius-marius-gib-mir-meine-legionen-wieder", "label": "table", "box": {"x1": 100, "y1": 100, "x2": 200, "y2": 200}}
+                    {
+                        "uuid": "marius-marius-gib-mir-meine-legionen-wieder",
+                        "label": "table",
+                        "box": {"x1": 100, "y1": 100, "x2": 200, "y2": 200},
+                    }
                 ],
             }
         ]