From b79bfd7db770552b94f1107a585e75f3f0464336 Mon Sep 17 00:00:00 2001 From: Matthias Bisping Date: Tue, 19 Apr 2022 17:58:19 +0200 Subject: [PATCH] containerized tests --- Dockerfile | 7 +----- Dockerfile_tests | 23 +++++++++++++++++++ .../main/resources/scripts/docker-build.sh | 4 ++-- .../image_extractor/extractors/parsable.py | 22 ++++++++---------- requirements.txt | 3 ++- run_tests.sh | 6 ++--- 6 files changed, 40 insertions(+), 25 deletions(-) create mode 100644 Dockerfile_tests diff --git a/Dockerfile b/Dockerfile index a8c37bd..635d9b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,10 @@ -ARG BASE_ROOT="nexus.iqser.com:5001/red/" -ARG VERSION_TAG="latest" - -FROM ${BASE_ROOT}image-prediction-base:${VERSION_TAG} +FROM image-prediction-base WORKDIR /app/service COPY src src COPY data data COPY image_prediction image_prediction -COPY incl/redai_image/redai incl/redai_image/redai COPY setup.py setup.py COPY requirements.txt requirements.txt COPY config.yaml config.yaml @@ -17,7 +13,6 @@ COPY config.yaml config.yaml RUN python3 -m pip install -r requirements.txt RUN python3 -m pip install -e . -RUN python3 -m pip install -e incl/redai_image/redai EXPOSE 5000 EXPOSE 8080 diff --git a/Dockerfile_tests b/Dockerfile_tests new file mode 100644 index 0000000..f9c00d2 --- /dev/null +++ b/Dockerfile_tests @@ -0,0 +1,23 @@ +ARG BASE_ROOT="nexus.iqser.com:5001/red/" +ARG VERSION_TAG="dev" + +FROM ${BASE_ROOT}image-prediction:${VERSION_TAG} + +WORKDIR /app/service + +COPY src src +COPY data data +COPY image_prediction image_prediction +COPY setup.py setup.py +COPY requirements.txt requirements.txt +COPY config.yaml config.yaml + +# Install module & dependencies +RUN python3 -m pip install -e . +RUN python3 -m pip install -r requirements.txt + +RUN apt update --yes +RUN apt install vim --yes +RUN apt install poppler-utils --yes + +CMD coverage run -m pytest test/ -x && coverage report -m && coverage xml diff --git a/bamboo-specs/src/main/resources/scripts/docker-build.sh b/bamboo-specs/src/main/resources/scripts/docker-build.sh index f17638f..54fe5f6 100755 --- a/bamboo-specs/src/main/resources/scripts/docker-build.sh +++ b/bamboo-specs/src/main/resources/scripts/docker-build.sh @@ -13,7 +13,7 @@ pip install 'dvc[ssh]' dvc pull echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf -docker build -f Dockerfile_base -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} . -docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} . +docker build -f Dockerfile_base -t $SERVICE_NAME_BASE . +docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} . echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} diff --git a/image_prediction/image_extractor/extractors/parsable.py b/image_prediction/image_extractor/extractors/parsable.py index dae6469..3448f6f 100644 --- a/image_prediction/image_extractor/extractors/parsable.py +++ b/image_prediction/image_extractor/extractors/parsable.py @@ -7,7 +7,7 @@ from typing import List import fitz from PIL import Image -from funcy import rcompose, merge, pluck, curry, compose +from funcy import rcompose, merge, pluck, curry, compose, rpartial from tqdm import tqdm from image_prediction.image_extractor.extractor import ImageExtractor, ImageMetadataPair @@ -35,20 +35,18 @@ class ParsablePDFImageExtractor(ImageExtractor): pages = extract_pages(self.doc, page_range) if page_range else self.doc - image_metadata_pairs = chain.from_iterable( - map( - self.__process_images_on_page, - tqdm( - pages, - desc=self.progress_message, - disable=not self.verbose, - total=len(page_range) if page_range else None, - ), - ) - ) + pages = self.__maybe_show_progress(pages, page_range) + + image_metadata_pairs = chain.from_iterable(map(self.__process_images_on_page, pages)) yield from image_metadata_pairs + def __maybe_show_progress(self, iterable, page_range): + return self.__progressbar(page_range)(iterable) if self.verbose else iterable + + def __progressbar(self, page_range): + return rpartial(tqdm, desc=self.progress_message, total=len(page_range) if page_range else None) + def __process_images_on_page(self, page: fitz.fitz.Page): images = get_images_on_page(self.doc, page) metadata = get_metadata_for_images_on_page(self.doc, page) diff --git a/requirements.txt b/requirements.txt index bbfa69a..8b33526 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,4 +19,5 @@ fpdf==1.7.2 coverage==6.3.2 Pillow==9.1.0 PDFNetPython3==9.1.0 -pdf2image==1.16.0 \ No newline at end of file +pdf2image==1.16.0 +frozendict==2.3.0 diff --git a/run_tests.sh b/run_tests.sh index 5ee1083..5cd92f3 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,11 +1,9 @@ echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 -docker build -f Dockerfile_tests -t image-prediction-tests:dev . -docker tag image-prediction-tests:dev nexus.iqser.com:5001/red/image-prediction-tests:dev -docker push nexus.iqser.com:5001/red/image-prediction-tests:dev +docker build -f Dockerfile_tests -t image-prediction-tests . rnd=$(date +"%s") name=image-prediction-tests-${rnd} echo "running tests container" -docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests:dev +docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests