containerized tests

Dockerfile

@@ -1,14 +1,10 @@
-ARG BASE_ROOT="nexus.iqser.com:5001/red/"
+FROM image-prediction-base
-ARG VERSION_TAG="latest"
-
-FROM ${BASE_ROOT}image-prediction-base:${VERSION_TAG}
 
 WORKDIR /app/service
 
 COPY src src
 COPY data data
 COPY image_prediction image_prediction
-COPY incl/redai_image/redai incl/redai_image/redai
 COPY setup.py setup.py
 COPY requirements.txt requirements.txt
 COPY config.yaml config.yaml
@@ -17,7 +13,6 @@ COPY config.yaml config.yaml
 RUN python3 -m pip install -r requirements.txt
 
 RUN python3 -m pip install -e .
-RUN python3 -m pip install -e incl/redai_image/redai
 
 EXPOSE 5000
 EXPOSE 8080

Dockerfile_tests

@@ -0,0 +1,23 @@
+ARG BASE_ROOT="nexus.iqser.com:5001/red/"
+ARG VERSION_TAG="dev"
+
+FROM ${BASE_ROOT}image-prediction:${VERSION_TAG}
+
+WORKDIR /app/service
+
+COPY src src
+COPY data data
+COPY image_prediction image_prediction
+COPY setup.py setup.py
+COPY requirements.txt requirements.txt
+COPY config.yaml config.yaml
+
+# Install module & dependencies
+RUN python3 -m pip install -e .
+RUN python3 -m pip install -r requirements.txt
+
+RUN apt update --yes
+RUN apt install vim --yes
+RUN apt install poppler-utils --yes
+
+CMD coverage run -m pytest test/ -x && coverage report -m && coverage xml

bamboo-specs/src/main/resources/scripts/docker-build.sh

@@ -13,7 +13,7 @@ pip install 'dvc[ssh]'
 dvc pull
 
 echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
-docker build -f Dockerfile_base  -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} .
+docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
-docker build -f Dockerfile  -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} .
+docker build -f Dockerfile  -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} .
 echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
 docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag}

image_prediction/image_extractor/extractors/parsable.py

@@ -7,7 +7,7 @@ from typing import List
 
 import fitz
 from PIL import Image
-from funcy import rcompose, merge, pluck, curry, compose
+from funcy import rcompose, merge, pluck, curry, compose, rpartial
 from tqdm import tqdm
 
 from image_prediction.image_extractor.extractor import ImageExtractor, ImageMetadataPair
@@ -35,20 +35,18 @@ class ParsablePDFImageExtractor(ImageExtractor):
 
         pages = extract_pages(self.doc, page_range) if page_range else self.doc
 
-        image_metadata_pairs = chain.from_iterable(
+        pages = self.__maybe_show_progress(pages, page_range)
-            map(
+
-                self.__process_images_on_page,
+        image_metadata_pairs = chain.from_iterable(map(self.__process_images_on_page, pages))
-                tqdm(
-                    pages,
-                    desc=self.progress_message,
-                    disable=not self.verbose,
-                    total=len(page_range) if page_range else None,
-                ),
-            )
-        )
 
         yield from image_metadata_pairs
 
+    def __maybe_show_progress(self, iterable, page_range):
+        return self.__progressbar(page_range)(iterable) if self.verbose else iterable
+
+    def __progressbar(self, page_range):
+        return rpartial(tqdm, desc=self.progress_message, total=len(page_range) if page_range else None)
+
     def __process_images_on_page(self, page: fitz.fitz.Page):
         images = get_images_on_page(self.doc, page)
         metadata = get_metadata_for_images_on_page(self.doc, page)

requirements.txt

@@ -20,3 +20,4 @@ coverage==6.3.2
 Pillow==9.1.0
 PDFNetPython3==9.1.0
 pdf2image==1.16.0
+frozendict==2.3.0

run_tests.sh

@@ -1,11 +1,9 @@
 echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
-docker build -f Dockerfile_tests -t image-prediction-tests:dev .
+docker build -f Dockerfile_tests -t image-prediction-tests .
-docker tag image-prediction-tests:dev nexus.iqser.com:5001/red/image-prediction-tests:dev
-docker push nexus.iqser.com:5001/red/image-prediction-tests:dev
 
 rnd=$(date +"%s")
 name=image-prediction-tests-${rnd}
 
 echo "running tests container"
 
-docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests:dev
+docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests