Marmelator/image-classification-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

containerized tests

Browse Source
This commit is contained in:
Matthias Bisping 2022-04-19 17:58:19 +02:00
parent 440698f706
commit b79bfd7db7
6 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Dockerfile
View File

@ -1,14 +1,10 @@
ARG BASE_ROOT="nexus.iqser.com:5001/red/"
ARG VERSION_TAG="latest"
FROM ${BASE_ROOT}image-prediction-base:${VERSION_TAG}
FROM image-prediction-base
WORKDIR /app/service
COPY src src
COPY data data
COPY image_prediction image_prediction
COPY incl/redai_image/redai incl/redai_image/redai
COPY setup.py setup.py
COPY requirements.txt requirements.txt
COPY config.yaml config.yaml
@ -17,7 +13,6 @@ COPY config.yaml config.yaml
RUN python3 -m pip install -r requirements.txt
RUN python3 -m pip install -e .
RUN python3 -m pip install -e incl/redai_image/redai
EXPOSE 5000
EXPOSE 8080

23
Dockerfile_tests Normal file
View File

@ -0,0 +1,23 @@
ARG BASE_ROOT="nexus.iqser.com:5001/red/"
ARG VERSION_TAG="dev"
FROM ${BASE_ROOT}image-prediction:${VERSION_TAG}
WORKDIR /app/service
COPY src src
COPY data data
COPY image_prediction image_prediction
COPY setup.py setup.py
COPY requirements.txt requirements.txt
COPY config.yaml config.yaml
# Install module & dependencies
RUN python3 -m pip install -e .
RUN python3 -m pip install -r requirements.txt
RUN apt update --yes
RUN apt install vim --yes
RUN apt install poppler-utils --yes
CMD coverage run -m pytest test/ -x && coverage report -m && coverage xml

4
bamboo-specs/src/main/resources/scripts/docker-build.sh
View File

@ -13,7 +13,7 @@ pip install 'dvc[ssh]'
dvc pull
echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
docker build -f Dockerfile_base -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} .
docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} .
docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} .
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag}

22
image_prediction/image_extractor/extractors/parsable.py
View File

@ -7,7 +7,7 @@ from typing import List
import fitz
from PIL import Image
from funcy import rcompose, merge, pluck, curry, compose
from funcy import rcompose, merge, pluck, curry, compose, rpartial
from tqdm import tqdm
from image_prediction.image_extractor.extractor import ImageExtractor, ImageMetadataPair
@ -35,20 +35,18 @@ class ParsablePDFImageExtractor(ImageExtractor):
pages = extract_pages(self.doc, page_range) if page_range else self.doc
image_metadata_pairs = chain.from_iterable(
map(
self.__process_images_on_page,
tqdm(
pages,
desc=self.progress_message,
disable=not self.verbose,
total=len(page_range) if page_range else None,
),
)
)
pages = self.__maybe_show_progress(pages, page_range)
image_metadata_pairs = chain.from_iterable(map(self.__process_images_on_page, pages))
yield from image_metadata_pairs
def __maybe_show_progress(self, iterable, page_range):
return self.__progressbar(page_range)(iterable) if self.verbose else iterable
def __progressbar(self, page_range):
return rpartial(tqdm, desc=self.progress_message, total=len(page_range) if page_range else None)
def __process_images_on_page(self, page: fitz.fitz.Page):
images = get_images_on_page(self.doc, page)
metadata = get_metadata_for_images_on_page(self.doc, page)

3
requirements.txt
View File

@ -19,4 +19,5 @@ fpdf==1.7.2
coverage==6.3.2
Pillow==9.1.0
PDFNetPython3==9.1.0
pdf2image==1.16.0
pdf2image==1.16.0
frozendict==2.3.0

6
run_tests.sh
View File

@ -1,11 +1,9 @@
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
docker build -f Dockerfile_tests -t image-prediction-tests:dev .
docker tag image-prediction-tests:dev nexus.iqser.com:5001/red/image-prediction-tests:dev
docker push nexus.iqser.com:5001/red/image-prediction-tests:dev
docker build -f Dockerfile_tests -t image-prediction-tests .
rnd=$(date +"%s")
name=image-prediction-tests-${rnd}
echo "running tests container"
docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests:dev
docker run --rm --net=host --name $name -v $PWD:$PWD -w $PWD -v /var/run/docker.sock:/var/run/docker.sock image-prediction-tests