Marmelator/cv-analysis-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 37 Wiki Activity

chore: repo housekeeping: adapt pre-commit and versioning script

Browse Source
This commit is contained in:
iriley 2024-04-29 13:20:14 +02:00
parent 5d13d8b3d0
commit b43033e6bf
21 changed files with 293 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.coveragerc
View File

@ -10,7 +10,7 @@ omit =
*/build_venv/*
*/incl/*
source =
cv_analysis
cv_analysis
relative_files = True
data_file = .coverage
@ -46,4 +46,4 @@ ignore_errors = True
directory = reports
[xml]
output = reports/coverage.xml
output = reports/coverage.xml

2
.dockerignore
View File

@ -97,4 +97,4 @@ target/
*.swp
*/*.swp
*/*/*.swp
*/*/*/*.swp
*/*/*/*.swp

3
.dvc/config
View File

@ -8,6 +8,3 @@
connection_string = "DefaultEndpointsProtocol=https;AccountName=cvsacricket;AccountKey=KOuTAQ6Mp00ePTT5ObYmgaHlxwS1qukY4QU4Kuk7gy/vldneA+ZiKjaOpEFtqKA6Mtym2gQz8THy+ASts/Y1Bw==;EndpointSuffix=core.windows.net"
['remote "local"']
url = ../dvc_local_remote

63
.hooks/poetry_version_check.py Normal file
View File

@ -0,0 +1,63 @@
import subprocess
import sys
from pathlib import Path
import semver
from loguru import logger
from semver.version import Version
logger.remove()
logger.add(sys.stdout, level="INFO")
def bashcmd(cmds: list) -> str:
try:
logger.debug(f"running: {' '.join(cmds)}")
return subprocess.run(
cmds, check=True, capture_output=True, text=True
).stdout.strip("\n")
except:
logger.warning(f"Error executing the following bash command: {' '.join(cmds)}.")
raise
def get_highest_existing_git_version_tag() -> str:
"""Get highest versions from git tags depending on bump level"""
try:
git_tags = bashcmd(["git", "tag", "-l"]).split()
semver_compat_tags = list(filter(Version.is_valid, git_tags))
highest_git_version_tag = max(semver_compat_tags, key=semver.version.Version.parse)
logger.info(f"Highest git version tag: {highest_git_version_tag}")
return highest_git_version_tag
except:
logger.warning("Error getting git version tags")
raise
def auto_bump_version() -> bool:
active = Path(".autoversion").is_file()
logger.debug(f"Automated version bump is set to '{active}'")
return active
def main() -> None:
poetry_project_version = bashcmd(["poetry", "version", "-s"])
logger.info(f"Poetry project version: {poetry_project_version}")
highest_git_version_tag = get_highest_existing_git_version_tag()
comparison_result = semver.compare(poetry_project_version, highest_git_version_tag)
if comparison_result in (-1, 0):
logger.warning("Poetry version must be greater than git tag version.")
if auto_bump_version():
logger.info(bashcmd(["poetry", "version", highest_git_version_tag]))
sys.exit(0)
sys.exit(1)
else:
logger.info(f"All good: {poetry_project_version} > {highest_git_version_tag}")
if __name__ == "__main__":
main()

71
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,71 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
exclude: ^(docs/|notebooks/|data/|src/configs/|tests/|.hooks/|bom.json)
default_language_version:
python: python3.10
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-toml
- id: detect-private-key
- id: check-added-large-files
args: ['--maxkb=10000']
- id: check-case-conflict
- id: mixed-line-ending
# - repo: https://github.com/pre-commit/mirrors-pylint
# rev: v3.0.0a5
# hooks:
# - id: pylint
# args:
# - --disable=C0111,R0903,E0401
# - --max-line-length=120
- repo: https://github.com/pre-commit/mirrors-isort
rev: v5.10.1
hooks:
- id: isort
args:
- --profile black
- repo: https://github.com/psf/black
rev: 24.3.0
hooks:
- id: black
# exclude: ^(docs/|notebooks/|data/|src/secrets/)
args:
- --line-length=120
- repo: https://github.com/compilerla/conventional-pre-commit
rev: v3.2.0
hooks:
- id: conventional-pre-commit
pass_filenames: false
stages: [commit-msg]
# args: [] # optional: list of Conventional Commits types to allow e.g. [feat, fix, ci, chore, test]
- repo: local
hooks:
- id: version-checker
name: version-checker
entry: python .hooks/poetry_version_check.py
language: python
always_run: true
additional_dependencies:
- "semver"
- "loguru"
# - repo: local
# hooks:
# - id: docker-build-test
# name: testing docker build
# entry: ./scripts/ops/docker-compose-build-run.sh
# language: script
# # always_run: true
# pass_filenames: false
# args: []
# stages: [pre-commit]

2
config/pyinfra.toml
View File

@ -41,4 +41,4 @@ connection_string = ""
[storage.tenant_server]
public_key = ""
endpoint = "http://tenant-user-management:8081/internal-api/tenants"
endpoint = "http://tenant-user-management:8081/internal-api/tenants"

2
docker-compose.yaml
View File

@ -28,4 +28,4 @@ services:
volumes:
- /opt/bitnami/rabbitmq/.rabbitmq/:/data/bitnami
volumes:
mdata:
mdata:

101
poetry.lock generated
View File

@ -677,6 +677,17 @@ files = [
[package.dependencies]
pycparser = "*"
[[package]]
name = "cfgv"
version = "3.4.0"
description = "Validate configuration and produce human readable error messages."
optional = false
python-versions = ">=3.8"
files = [
{file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
{file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
]
[[package]]
name = "charset-normalizer"
version = "3.3.2"
@ -1142,6 +1153,17 @@ files = [
{file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"},
]
[[package]]
name = "distlib"
version = "0.3.8"
description = "Distribution utilities"
optional = false
python-versions = "*"
files = [
{file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"},
{file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"},
]
[[package]]
name = "distro"
version = "1.9.0"
@ -1973,6 +1995,20 @@ antlr4-python3-runtime = "==4.9.*"
omegaconf = ">=2.2,<2.4"
packaging = "*"
[[package]]
name = "identify"
version = "2.5.36"
description = "File identification library for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"},
{file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"},
]
[package.extras]
license = ["ukkonen"]
[[package]]
name = "idna"
version = "3.7"
@ -2855,6 +2891,20 @@ doc = ["myst-nb (>=1.0)", "numpydoc (>=1.7)", "pillow (>=9.4)", "pydata-sphinx-t
extra = ["lxml (>=4.6)", "pydot (>=2.0)", "pygraphviz (>=1.12)", "sympy (>=1.10)"]
test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"]
[[package]]
name = "nodeenv"
version = "1.8.0"
description = "Node.js virtual environment builder"
optional = false
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
files = [
{file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"},
{file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"},
]
[package.dependencies]
setuptools = "*"
[[package]]
name = "numpy"
version = "1.26.4"
@ -3517,6 +3567,24 @@ docs = ["sphinx (>=1.7.1)"]
redis = ["redis"]
tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=6.0.0)", "types-redis"]
[[package]]
name = "pre-commit"
version = "3.7.0"
description = "A framework for managing and maintaining multi-language pre-commit hooks."
optional = false
python-versions = ">=3.9"
files = [
{file = "pre_commit-3.7.0-py2.py3-none-any.whl", hash = "sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab"},
{file = "pre_commit-3.7.0.tar.gz", hash = "sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060"},
]
[package.dependencies]
cfgv = ">=2.0.0"
identify = ">=1.0.0"
nodeenv = ">=0.11.1"
pyyaml = ">=5.1"
virtualenv = ">=20.10.0"
[[package]]
name = "prometheus-client"
version = "0.18.0"
@ -4361,6 +4429,17 @@ shortuuid = ">=0.5.0"
dev = ["mock (==5.1.0)", "mypy (==0.971)", "paramiko (==3.3.1)", "pylint (==2.15.0)", "pytest (==7.2.0)", "pytest-asyncio (==0.18.3)", "pytest-cov (==3.0.0)", "pytest-docker (==0.12.0)", "pytest-mock (==3.8.2)", "pytest-sugar (==0.9.5)", "pytest-test-utils (==0.0.8)", "types-certifi (==2021.10.8.3)", "types-mock (==5.1.0.2)", "types-paramiko (==3.3.0.0)"]
tests = ["mock (==5.1.0)", "mypy (==0.971)", "paramiko (==3.3.1)", "pylint (==2.15.0)", "pytest (==7.2.0)", "pytest-asyncio (==0.18.3)", "pytest-cov (==3.0.0)", "pytest-docker (==0.12.0)", "pytest-mock (==3.8.2)", "pytest-sugar (==0.9.5)", "pytest-test-utils (==0.0.8)", "types-certifi (==2021.10.8.3)", "types-mock (==5.1.0.2)", "types-paramiko (==3.3.0.0)"]
[[package]]
name = "semver"
version = "3.0.2"
description = "Python helper for Semantic Versioning (https://semver.org)"
optional = false
python-versions = ">=3.7"
files = [
{file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"},
{file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"},
]
[[package]]
name = "setuptools"
version = "69.5.1"
@ -4669,6 +4748,26 @@ files = [
{file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
]
[[package]]
name = "virtualenv"
version = "20.26.0"
description = "Virtual Python Environment builder"
optional = false
python-versions = ">=3.7"
files = [
{file = "virtualenv-20.26.0-py3-none-any.whl", hash = "sha256:0846377ea76e818daaa3e00a4365c018bc3ac9760cbb3544de542885aad61fb3"},
{file = "virtualenv-20.26.0.tar.gz", hash = "sha256:ec25a9671a5102c8d2657f62792a27b48f016664c6873f6beed3800008577210"},
]
[package.dependencies]
distlib = ">=0.3.7,<1"
filelock = ">=3.12.2,<4"
platformdirs = ">=3.9.1,<5"
[package.extras]
docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"]
[[package]]
name = "voluptuous"
version = "0.14.2"
@ -4954,4 +5053,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
content-hash = "06b9635bd0acdb0cd78ee9ab0e6a8c75ef91648c36bbc1f78ac44a5c0671990b"
content-hash = "971cfd0d8034dcf0cefd68d8fff173a36a83fe42625b5eca31d4b5275fbc8e1f"

10
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "cv-analysis-service"
version = "2.1.0"
version = "2.5.1"
description = ""
authors = []
readme = "README.md"
@ -41,6 +41,8 @@ pylint = "^3.1"
ipython = "^8.21.0"
mypy = "^1.10.0"
pylint = "^3.1.0"
pre-commit = "^3.7.0"
semver = "^3.0.2"
[tool.pytest.ini_options]
testpaths = ["test"]
@ -66,6 +68,12 @@ docstring-min-length=4
extension-pkg-whitelist = ["cv2"]
extension-pkg-allow-list = ["cv2"]
[tool.black]
line-length = 120
[tool.isort]
profile = "black"
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

4
scripts/annotate.py
View File

@ -45,7 +45,7 @@ if __name__ == "__main__":
elif args.type == "layout":
from cv_analysis.layout_parsing import parse_layout as analyze
elif args.type == "figure":
from cv_analysis.figure_detection.figure_detection import \
detect_figures
from cv_analysis.figure_detection.figure_detection import detect_figures
analyze = detect_figures
annotate_page(page, analyze, draw, name=name, show=args.show)

7
scripts/annotate_pdf.py
View File

@ -56,7 +56,6 @@ if __name__ == "__main__":
t2 = timeit.default_timer()
save_as_pdf(annotated_pages, args.output_folder, Path(args.pdf_path).stem, args.type)
t3 = timeit.default_timer()
print("[s] opening file and convert pdf pages to images: ", t1-t0)
print("[s] analyse and annotate images: ", t2-t1)
print("[s] save images as pdf: ", t3-t2)
print("[s] opening file and convert pdf pages to images: ", t1 - t0)
print("[s] analyse and annotate images: ", t2 - t1)
print("[s] save images as pdf: ", t3 - t2)

2
scripts/docker_build_run.sh
View File

@ -1,4 +1,4 @@
docker build -t cv-analysis-service:$(poetry version -s)-dev \
docker build -t cv-analysis-service:$(poetry version -s)-dev \
-f Dockerfile \
--build-arg USERNAME=$GITLAB_USER \
--build-arg TOKEN=$GITLAB_ACCESS_TOKEN \

2
scripts/explore_aio_detection.py
View File

@ -94,7 +94,7 @@ def classify_node_recursively(image, node):
if maybe_texts:
children = lmap(lambda b: make_child(b, node, ["text"]), maybe_texts)
node.children = children
return node #FIGURES
return node # FIGURES
return node

6
src/cv_analysis/figure_detection/figure_detection.py
View File

@ -5,7 +5,11 @@ import numpy as np
from cv_analysis.figure_detection.figures import detect_large_coherent_structures
from cv_analysis.figure_detection.text import remove_primary_text_regions
from cv_analysis.utils.filters import has_acceptable_format, is_large_enough, is_not_too_large
from cv_analysis.utils.filters import (
has_acceptable_format,
is_large_enough,
is_not_too_large,
)
from cv_analysis.utils.postprocessing import remove_included
from cv_analysis.utils.structures import Rectangle

6
src/cv_analysis/layout_parsing.py
View File

@ -6,7 +6,11 @@ import cv2
import numpy as np
from cv_analysis.utils.connect_rects import connect_related_rects2
from cv_analysis.utils.postprocessing import has_no_parent, remove_included, remove_overlapping
from cv_analysis.utils.postprocessing import (
has_no_parent,
remove_included,
remove_overlapping,
)
from cv_analysis.utils.structures import Rectangle
from cv_analysis.utils.visual_logging import vizlogger

8
src/cv_analysis/server/pipeline.py
View File

@ -10,7 +10,10 @@ from pdf2img.default_objects.rectangle import RectanglePlus
from cv_analysis.figure_detection.figure_detection import detect_figures
from cv_analysis.table_inference import infer_lines
from cv_analysis.table_parsing import parse_lines, parse_tables
from cv_analysis.utils.image_extraction import extract_images_from_pdf, transform_table_lines_by_page_info
from cv_analysis.utils.image_extraction import (
extract_images_from_pdf,
transform_table_lines_by_page_info,
)
from cv_analysis.utils.structures import Rectangle
@ -49,8 +52,7 @@ def make_image_analysis_pipeline(
# rel_bboxes = map()
img_results = lmap(analysis_fn, images)
def make_offsets():
...
def make_offsets(): ...
offsets = map(itemgetter("x1", "y2"), map(itemgetter("bbox"), info))
# print("before", img_results)

4
src/cv_analysis/table_parsing.py
View File

@ -3,7 +3,9 @@ from cv2 import cv2
from funcy import lfilter, lmap # type: ignore
from cv_analysis.layout_parsing import parse_layout
from cv_analysis.utils.postprocessing import remove_isolated # xywh_to_vecs, xywh_to_vec_rect, adjacent1d
from cv_analysis.utils.postprocessing import (
remove_isolated, # xywh_to_vecs, xywh_to_vec_rect, adjacent1d
)
from cv_analysis.utils.structures import Rectangle
from cv_analysis.utils.visual_logging import vizlogger

10
src/cv_analysis/utils/banner.py
View File

@ -1,13 +1,13 @@
def make_art():
art = r"""
__
_ |@@|
__
_ |@@|
/ \ \--/ __ .__ .__
) O|----| | __ ___ __ _____ ____ _____ | | ___.__. _____|__| ______
/ / \ }{ /\ )_ / _\\ \/ / ______ \__ \ / \\__ \ | | | | |/ ___/ |/ ___/
)/ /\__/\ \__O (__ \ / /_____/ / __ \| | \/ __ \| |_\___ |\___ \| |\___ \
|/ (--/\--) \__/ \_/ (______/___|__(______/____/\____/_____/|__/_____/
/ _)( )(_
`---''---`
|/ (--/\--) \__/ \_/ (______/___|__(______/____/\____/_____/|__/_____/
/ _)( )(_
`---''---`
"""
return art

6
test/unit_tests/figure_detection/text_test.py
View File

@ -4,8 +4,10 @@ import cv2
import numpy as np
import pytest
from cv_analysis.figure_detection.text import (apply_threshold_to_image,
remove_primary_text_regions)
from cv_analysis.figure_detection.text import (
apply_threshold_to_image,
remove_primary_text_regions,
)
@pytest.mark.parametrize("error_tolerance", [0.07])

12
test/unit_tests/server_pipeline_test.py
View File

@ -2,9 +2,11 @@ import fitz
import numpy as np
import pytest
from cv_analysis.server.pipeline import (figure_detection_formatter,
make_analysis_pipeline,
table_parsing_formatter)
from cv_analysis.server.pipeline import (
figure_detection_formatter,
make_analysis_pipeline,
table_parsing_formatter,
)
from cv_analysis.utils.structures import Rectangle
@ -78,8 +80,6 @@ def formatter(operation):
@pytest.mark.parametrize("operation", ["figure"])
def test_analysis_pipeline(empty_pdf, formatter, expected_formatted_analysis_result):
analysis_pipeline = make_analysis_pipeline(
analysis_fn_mock, formatter, dpi=200, skip_pages_without_images=False
)
analysis_pipeline = make_analysis_pipeline(analysis_fn_mock, formatter, dpi=200, skip_pages_without_images=False)
results = list(analysis_pipeline(empty_pdf))
assert list(results) == expected_formatted_analysis_result

6
test/unit_tests/table_inference_test.py
View File

@ -11,7 +11,11 @@ def test_table_inference_smoke():
{
"page_idx": 1,
"boxes": [
{"uuid": "marius-marius-gib-mir-meine-legionen-wieder", "label": "table", "box": {"x1": 100, "y1": 100, "x2": 200, "y2": 200}}
{
"uuid": "marius-marius-gib-mir-meine-legionen-wieder",
"label": "table",
"box": {"x1": 100, "y1": 100, "x2": 200, "y2": 200},
}
],
}
]