update project structure
This commit is contained in:
Francisco Schulz
parent
25fc7d84b9
commit
8cce535301
@ -103,4 +103,4 @@ target/
|
||||
*.swp
|
||||
*/*.swp
|
||||
*/*/*.swp
|
||||
*/*/*/*.swp
|
||||
*/*/*/*.swp
|
||||
|
||||
26
.pre-commit-config.yaml
Normal file
26
.pre-commit-config.yaml
Normal file
@ -0,0 +1,26 @@
|
||||
# See https://pre-commit.com for more information
|
||||
# See https://pre-commit.com/hooks.html for more hooks
|
||||
exclude: ^(docs/|notebooks/|data/|src/secrets/|src/static/|src/templates/|tests)
|
||||
default_language_version:
|
||||
python: python3.8
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.4.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-isort
|
||||
rev: v5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
args: ["--profile", "black"]
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 23.1.0
|
||||
hooks:
|
||||
- id: black
|
||||
# exclude: ^(docs/|notebooks/|data/|src/secrets/)
|
||||
args:
|
||||
- --line-length=120
|
||||
1
.python-version
Normal file
1
.python-version
Normal file
@ -0,0 +1 @@
|
||||
3.8.13
|
||||
52
Dockerfile
52
Dockerfile
@ -1,27 +1,43 @@
|
||||
FROM image-prediction-base
|
||||
FROM python:3.8 as builder
|
||||
ARG SSH_KEY
|
||||
# ARG SSH_KEY_PUB
|
||||
ENV PYTHONUNBUFFERED=true
|
||||
WORKDIR /app
|
||||
|
||||
WORKDIR /app/service
|
||||
ENV POETRY_HOME=/opt/poetry
|
||||
ENV POETRY_VIRTUALENVS_IN_PROJECT=true
|
||||
ENV PATH="$POETRY_HOME/bin:$PATH"
|
||||
|
||||
COPY src src
|
||||
COPY incl/pyinfra incl/pyinfra
|
||||
COPY incl/pdf2image incl/pdf2image
|
||||
COPY data data
|
||||
COPY image_prediction image_prediction
|
||||
COPY setup.py setup.py
|
||||
COPY requirements.txt requirements.txt
|
||||
COPY config.yaml config.yaml
|
||||
COPY banner.txt banner.txt
|
||||
RUN mkdir /root/.ssh/ && \
|
||||
chmod 700 /root/.ssh/ && \
|
||||
echo "${SSH_KEY}" > /root/.ssh/id_rsa && \
|
||||
chmod 600 /root/.ssh/id_rsa && \
|
||||
ssh-keyscan -p 2222 git.iqser.com >> /root/.ssh/known_hosts
|
||||
# echo "${SSH_KEY_PUB}" > /root/.ssh/id_rsa.pub && \
|
||||
# chmod 600 /root/.ssh/id_rsa.pub && \
|
||||
|
||||
# Install dependencies differing from base image.
|
||||
RUN python3 -m pip install -r requirements.txt
|
||||
RUN python3 -m pip install -r incl/pyinfra/requirements.txt
|
||||
RUN python3 -m pip install -r incl/pdf2image/requirements.txt
|
||||
RUN curl -sSL https://install.python-poetry.org | python3 -
|
||||
|
||||
RUN python3 -m pip install -e .
|
||||
RUN python3 -m pip install -e incl/pyinfra
|
||||
RUN python3 -m pip install -e incl/pdf2image
|
||||
COPY pyproject.toml poetry.lock ./
|
||||
RUN poetry config virtualenvs.create false && \
|
||||
poetry install --without=dev -vvv --no-interaction --no-root
|
||||
|
||||
FROM python:3.8-bullseye as prod
|
||||
ENV PYTHONUNBUFFERED=true
|
||||
WORKDIR /app
|
||||
ENV PATH="/app/.venv/bin:$PATH"
|
||||
COPY --from=builder /app /app
|
||||
|
||||
COPY ./src ./
|
||||
USER 1001
|
||||
|
||||
EXPOSE 5000
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["python3", "src/serve.py"]
|
||||
|
||||
# TESTS
|
||||
# RUN apt update --yes
|
||||
# RUN apt install vim --yes
|
||||
# RUN apt install poppler-utils --yes
|
||||
# CMD coverage run -m pytest test/ --tb=native -q -s -vvv -x && coverage combine && coverage report -m && coverage xml
|
||||
|
||||
@ -1,25 +0,0 @@
|
||||
FROM python:3.8 as builder1
|
||||
|
||||
# Use a virtual environment.
|
||||
RUN python -m venv /app/venv
|
||||
ENV PATH="/app/venv/bin:$PATH"
|
||||
|
||||
# Upgrade pip.
|
||||
RUN python -m pip install --upgrade pip
|
||||
|
||||
# Make a directory for the service files and copy the service repo into the container.
|
||||
WORKDIR /app/service
|
||||
COPY ./requirements.txt ./requirements.txt
|
||||
|
||||
# Install dependencies.
|
||||
RUN python3 -m pip install -r requirements.txt
|
||||
|
||||
# Make a new container and copy all relevant files over to filter out temporary files
|
||||
# produced during setup to reduce the final container's size.
|
||||
FROM python:3.8
|
||||
|
||||
WORKDIR /app/
|
||||
COPY --from=builder1 /app .
|
||||
ENV PATH="/app/venv/bin:$PATH"
|
||||
|
||||
WORKDIR /app/service
|
||||
@ -1,31 +0,0 @@
|
||||
ARG BASE_ROOT="nexus.iqser.com:5001/red/"
|
||||
ARG VERSION_TAG="dev"
|
||||
|
||||
FROM ${BASE_ROOT}image-prediction:${VERSION_TAG}
|
||||
|
||||
WORKDIR /app/service
|
||||
|
||||
COPY src src
|
||||
COPY incl/pyinfra incl/pyinfra
|
||||
COPY incl/pdf2image incl/pdf2image
|
||||
COPY data data
|
||||
COPY image_prediction image_prediction
|
||||
COPY setup.py setup.py
|
||||
COPY requirements.txt requirements.txt
|
||||
COPY config.yaml config.yaml
|
||||
COPY banner.txt banner.txt
|
||||
|
||||
# Install module & dependencies
|
||||
RUN python3 -m pip install -r requirements.txt
|
||||
RUN python3 -m pip install -r incl/pyinfra/requirements.txt
|
||||
RUN python3 -m pip install -r incl/pdf2image/requirements.txt
|
||||
|
||||
RUN python3 -m pip install -e .
|
||||
RUN python3 -m pip install -e incl/pyinfra
|
||||
RUN python3 -m pip install -e incl/pdf2image
|
||||
|
||||
RUN apt update --yes
|
||||
RUN apt install vim --yes
|
||||
RUN apt install poppler-utils --yes
|
||||
|
||||
CMD coverage run -m pytest test/ --tb=native -q -s -vvv -x && coverage combine && coverage report -m && coverage xml
|
||||
88
Makefile
Normal file
88
Makefile
Normal file
@ -0,0 +1,88 @@
|
||||
.PHONY: \
|
||||
poetry in-project-venv dev-env use-env install install-dev tests \
|
||||
update-version sync-version-with-git \
|
||||
docker docker-build-run docker-build docker-run \
|
||||
docker-rm docker-rm-container docker-rm-image \
|
||||
pre-commit get-licenses prep-commit \
|
||||
docs sphinx_html sphinx_apidoc
|
||||
.DEFAULT_GOAL := run
|
||||
|
||||
export DOCKER=docker
|
||||
export DOCKERFILE=Dockerfile
|
||||
export IMAGE_NAME=imgage-prediction-image
|
||||
export CONTAINER_NAME=imgage-prediction-container
|
||||
export HOST_PORT=9999
|
||||
export CONTAINER_PORT=9999
|
||||
export PYTHON_VERSION=python3.8
|
||||
export SSH_KEY=$(cat ~/.ssh/id_ecdsa)
|
||||
|
||||
# all commands should be executed in the root dir or the project,
|
||||
# specific environments should be deactivated
|
||||
|
||||
poetry: in-project-venv use-env dev-env
|
||||
|
||||
in-project-venv:
|
||||
poetry config virtualenvs.in-project true
|
||||
|
||||
use-env:
|
||||
poetry env use ${PYTHON_VERSION}
|
||||
|
||||
dev-env:
|
||||
poetry install --with dev
|
||||
|
||||
install:
|
||||
poetry add $(pkg)
|
||||
|
||||
install-dev:
|
||||
poetry add --dev $(pkg)
|
||||
|
||||
requirements:
|
||||
poetry export --without-hashes --output ./src/requirements.txt
|
||||
|
||||
update-version:
|
||||
poetry version prerelease
|
||||
|
||||
sync-version-with-git:
|
||||
git pull -p && poetry version $(git rev-list --tags --max-count=1 | git describe --tags --abbrev=0)
|
||||
|
||||
docker: docker-rm docker-build-run
|
||||
|
||||
docker-build-run: docker-build docker-run
|
||||
|
||||
docker-build:
|
||||
$(DOCKER) build \
|
||||
--no-cache --progress=plain \
|
||||
--build-arg SSH_KEY=$(SSH_KEY) \
|
||||
-t $(IMAGE_NAME) \
|
||||
-f $(DOCKERFILE) .
|
||||
|
||||
docker-run:
|
||||
$(DOCKER) run -it --rm -p $(HOST_PORT):$(CONTAINER_PORT)/tcp --name $(CONTAINER_NAME) $(IMAGE_NAME) python ./src/serve.py
|
||||
|
||||
docker-rm: docker-rm-container docker-rm-image
|
||||
|
||||
docker-rm-container:
|
||||
-$(DOCKER) rm $(CONTAINER_NAME)
|
||||
|
||||
docker-rm-image:
|
||||
-$(DOCKER) image rm $(IMAGE_NAME)
|
||||
|
||||
tests:
|
||||
poetry run pytest ./tests
|
||||
|
||||
prep-commit:
|
||||
docs get-license sync-version-with-git update-version pre-commit
|
||||
|
||||
pre-commit:
|
||||
pre-commit run --all-files
|
||||
|
||||
get-licenses:
|
||||
pip-licenses --format=json --order=license --with-urls > pkg-licenses.json
|
||||
|
||||
docs: sphinx_apidoc sphinx_html
|
||||
|
||||
sphinx_html:
|
||||
poetry run sphinx-build -b html docs/source/ docs/build/html -E -a
|
||||
|
||||
sphinx_apidoc:
|
||||
poetry run sphinx-apidoc -o ./docs/source/modules ./src/rule_engine
|
||||
22
README.md
22
README.md
@ -34,7 +34,7 @@ python scripts/pyinfra_mock.py /path/to/a/pdf
|
||||
Run for example this command to execute all tests and get a coverage report:
|
||||
|
||||
```bash
|
||||
coverage run -m pytest test --tb=native -q -s -vvv -x && coverage combine && coverage report -m
|
||||
coverage run -m pytest tests --tb=native -q -s -vvv -x && coverage combine && coverage report -m
|
||||
```
|
||||
|
||||
After having built the service container as specified above, you can also run tests in a container as follows:
|
||||
@ -126,15 +126,15 @@ An image metadata record (entry in `"data"` field of a response body) looks like
|
||||
A configuration file is located under `config.yaml`. All relevant variables can be configured via
|
||||
exporting environment variables.
|
||||
|
||||
| __Environment Variable__ | Default | Description |
|
||||
|------------------------------------|------------------------------------|----------------------------------------------------------------------------------------|
|
||||
| __LOGGING_LEVEL_ROOT__ | "INFO" | Logging level for log file messages |
|
||||
| __VERBOSE__ | *true* | Service prints document processing progress to stdout |
|
||||
| __BATCH_SIZE__ | 16 | Number of images in memory simultaneously per service instance |
|
||||
| __RUN_ID__ | "fabfb1f192c745369b88cab34471aba7" | The ID of the mlflow run to load the image classifier from |
|
||||
| __MIN_REL_IMAGE_SIZE__ | 0.05 | Minimally permissible image size to page size ratio |
|
||||
| __MAX_REL_IMAGE_SIZE__ | 0.75 | Maximally permissible image size to page size ratio |
|
||||
| __MIN_IMAGE_FORMAT__ | 0.1 | Minimally permissible image width to height ratio |
|
||||
| __MAX_IMAGE_FORMAT__ | 10 | Maximally permissible image width to height ratio |
|
||||
| __Environment Variable__ | Default | Description |
|
||||
| ------------------------ | ---------------------------------- | -------------------------------------------------------------- |
|
||||
| __LOGGING_LEVEL_ROOT__ | "INFO" | Logging level for log file messages |
|
||||
| __VERBOSE__ | *true* | Service prints document processing progress to stdout |
|
||||
| __BATCH_SIZE__ | 16 | Number of images in memory simultaneously per service instance |
|
||||
| __RUN_ID__ | "fabfb1f192c745369b88cab34471aba7" | The ID of the mlflow run to load the image classifier from |
|
||||
| __MIN_REL_IMAGE_SIZE__ | 0.05 | Minimally permissible image size to page size ratio |
|
||||
| __MAX_REL_IMAGE_SIZE__ | 0.75 | Maximally permissible image size to page size ratio |
|
||||
| __MIN_IMAGE_FORMAT__ | 0.1 | Minimally permissible image width to height ratio |
|
||||
| __MAX_IMAGE_FORMAT__ | 10 | Maximally permissible image width to height ratio |
|
||||
|
||||
See also: https://git.iqser.com/projects/RED/repos/helm/browse/redaction/templates/image-service-v2
|
||||
|
||||
@ -37,4 +37,4 @@
|
||||
|
||||
<!-- run 'mvn test' to perform offline validation of the plan -->
|
||||
<!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
|
||||
</project>
|
||||
</project>
|
||||
|
||||
@ -33,21 +33,22 @@ import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
|
||||
|
||||
/**
|
||||
* Plan configuration for Bamboo.
|
||||
* Learn more on: <a href="https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
|
||||
* Learn more on: <a href=
|
||||
*
|
||||
* "https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
|
||||
*/
|
||||
@BambooSpec
|
||||
public class PlanSpec {
|
||||
|
||||
private static final String SERVICE_NAME = "image-prediction";
|
||||
private static final String SERVICE_NAME_BASE = "image-prediction-base";
|
||||
|
||||
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-","").replaceAll("_","");
|
||||
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "").replaceAll("_", "");
|
||||
|
||||
/**
|
||||
* Run main to publish plan on Bamboo
|
||||
*/
|
||||
public static void main(final String[] args) throws Exception {
|
||||
//By default credentials are read from the '.credentials' file.
|
||||
public static void main(final String[] args) throws Exception {
|
||||
// By default credentials are read from the '.credentials' file.
|
||||
BambooServer bambooServer = new BambooServer("http://localhost:8085");
|
||||
|
||||
Plan plan = new PlanSpec().createDockerBuildPlan();
|
||||
@ -63,10 +64,18 @@ public class PlanSpec {
|
||||
|
||||
private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
|
||||
Permissions permission = new Permissions()
|
||||
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
.groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD)
|
||||
|
||||
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN,
|
||||
|
||||
PermissionType.CLONE, PermissionType.BUILD)
|
||||
|
||||
.groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.
|
||||
LONE,
|
||||
PermissionType.BUILD)
|
||||
.groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE,
|
||||
PermissionType.BUILD)
|
||||
.groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE,
|
||||
PermissionType.BUILD)
|
||||
.loggedInUserPermissions(PermissionType.VIEW)
|
||||
.anonymousUserPermissionView();
|
||||
return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission);
|
||||
@ -79,102 +88,110 @@ public class PlanSpec {
|
||||
}
|
||||
|
||||
public Plan createDockerBuildPlan() {
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME, new BambooKey(SERVICE_KEY))
|
||||
.description("Docker build for image-prediction.")
|
||||
.stages(
|
||||
new Stage("Build Stage")
|
||||
.jobs(
|
||||
new Job("Build Job", new BambooKey("BUILD"))
|
||||
.tasks(
|
||||
new CleanWorkingDirectoryTask()
|
||||
.description("Clean working directory.")
|
||||
.enabled(true),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout default repository.")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Set config and keys.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
|
||||
new ScriptTask()
|
||||
.description("Build Docker container.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh")
|
||||
.argument(SERVICE_NAME + " " + SERVICE_NAME_BASE),
|
||||
new InjectVariablesTask()
|
||||
.description("Inject git tag.")
|
||||
.path("git.tag")
|
||||
.namespace("g")
|
||||
.scope(InjectVariablesScope.LOCAL),
|
||||
new VcsTagTask()
|
||||
.description("${bamboo.g.gitTag}")
|
||||
.tagName("${bamboo.g.gitTag}")
|
||||
.defaultRepository())
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/release_build:4.5.0")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock")),
|
||||
new Job("Licence Job", new BambooKey("LICENCE"))
|
||||
.enabled(false)
|
||||
.tasks(
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout default repository.")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Build licence.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh"))
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
|
||||
.volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RR / " + SERVICE_NAME)
|
||||
.linkedRepositories("RR / redai_image")
|
||||
.triggers(
|
||||
new BitbucketServerTrigger())
|
||||
.planBranchManagement(
|
||||
new PlanBranchManagement()
|
||||
.createForVcsBranch()
|
||||
.delete(
|
||||
new BranchCleanup()
|
||||
.whenInactiveInRepositoryAfterDays(14))
|
||||
.notificationForCommitters());
|
||||
}
|
||||
return new Plan(
|
||||
project(),
|
||||
SERVICE_NAME, new BambooKey(SERVICE_KEY))
|
||||
.description("Docker build for image-prediction.")
|
||||
.stages(
|
||||
new Stage("Build Stage")
|
||||
.jobs(
|
||||
new Job("Build Job", new BambooKey("BUILD"))
|
||||
.tasks(
|
||||
new CleanWorkingDirectoryTask()
|
||||
.description("Clean working directory.")
|
||||
.enabled(true),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout default repository.")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Set config and keys.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath(
|
||||
"bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
|
||||
new ScriptTask()
|
||||
.description("Build Docker container.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath(
|
||||
"bamboo-specs/src/main/resources/scripts/docker-build.sh")
|
||||
.argument(SERVICE_NAME),
|
||||
new InjectVariablesTask()
|
||||
.description("Inject git tag.")
|
||||
.path("git.tag")
|
||||
.namespace("g")
|
||||
.scope(InjectVariablesScope.LOCAL),
|
||||
new VcsTagTask()
|
||||
.description("${bamboo.g.gitTag}")
|
||||
.tagName("${bamboo.g.gitTag}")
|
||||
.defaultRepository())
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/release_build:4.5.0")
|
||||
.volume("/var/run/docker.sock",
|
||||
"/var/run/docker.sock")),
|
||||
new Job("Licence Job", new BambooKey("LICENCE"))
|
||||
.enabled(false)
|
||||
.tasks(
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout default repository.")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Build licence.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath(
|
||||
"bamboo-specs/src/main/resources/scripts/create-licence.sh"))
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0")
|
||||
.volume("/etc/maven/settings.xml",
|
||||
"/usr/share/maven/ref/settings.xml")
|
||||
.volume("/var/run/docker.sock",
|
||||
"/var/run/docker.sock"))))
|
||||
.linkedRepositories("RR / " + SERVICE_NAME)
|
||||
.linkedRepositories("RR / redai_image")
|
||||
.triggers(
|
||||
|
||||
public Plan createSecBuild() {
|
||||
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan")
|
||||
.stages(new Stage("Default Stage").jobs(
|
||||
new Job("Sonar Job", new BambooKey("SONAR"))
|
||||
.tasks(
|
||||
new CleanWorkingDirectoryTask()
|
||||
.description("Clean working directory.")
|
||||
.enabled(true),
|
||||
new VcsCheckoutTask()
|
||||
.description("Checkout default repository.")
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
.description("Set config and keys.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
|
||||
new ScriptTask()
|
||||
.description("Run Sonarqube scan.")
|
||||
.location(Location.FILE)
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh")
|
||||
.argument(SERVICE_NAME))
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/release_build:4.2.0")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
new BitbucketServerTrigger())
|
||||
nBranchManagement(
|
||||
new P
|
||||
.createForVcsBranch()
|
||||
.delete(
|
||||
new
|
||||
|
||||
.notificationForCommitters());
|
||||
|
||||
|
||||
ld() {
|
||||
ect(), SERVICE_NAME + "-
|
||||
("Security Analysis Plan")
|
||||
new Stage("Defau
|
||||
Job("Sonar Job", new BambooKey("SON
|
||||
.tasks(
|
||||
new CleanWorkingDirectoryTask()
|
||||
.des
|
||||
|
||||
new VcsChec
|
||||
.description("Checkout default repos
|
||||
.checkoutItems(new CheckoutItem().defaultRepository()),
|
||||
new ScriptTask()
|
||||
|
||||
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"),
|
||||
|
||||
|
||||
.location(Locati
|
||||
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh")
|
||||
.argument(SERVICE_NAME))
|
||||
.dockerConfiguration(
|
||||
new DockerConfiguration()
|
||||
.image("nexus.iqser.com:5001/infra/release_build:4.2.0")
|
||||
.volume("/var/run/docker.sock", "/var/run/docker.sock"))))
|
||||
.linkedRepositories("RR / " + SERVICE_NAME)
|
||||
.triggers(
|
||||
new ScheduledTrigger()
|
||||
.scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
new ScheduledTrigger()
|
||||
.scheduleOnceDaily(LocalTime.of(23, 00)))
|
||||
.planBranchManagement(
|
||||
new PlanBranchManagement()
|
||||
.createForVcsBranchMatching("release.*")
|
||||
.notificationForCommitters());
|
||||
new PlanBranchManagement()
|
||||
.createForVcsBranchMatching("release.*")
|
||||
.notificationForCommitters());
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,4 +16,4 @@ then
|
||||
-Dmaven.wagon.http.ssl.allowall=true \
|
||||
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
|
||||
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -2,55 +2,97 @@
|
||||
set -e
|
||||
|
||||
SERVICE_NAME=$1
|
||||
SERVICE_NAME_BASE=$2
|
||||
if [[ "$bamboo_planRepository_branchName" == "master" ]]
|
||||
then
|
||||
branchVersion=$(cat version.yaml | grep -Eo "version: .*" | sed -s 's|version: \(.*\)\..*\..*|\1|g')
|
||||
latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
|
||||
newVersion="$(semver $latestVersion -p -i minor)"
|
||||
echo "new release on master with version $newVersion"
|
||||
elif [[ "$bamboo_planRepository_branchName" == release* ]]
|
||||
then
|
||||
branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|')
|
||||
latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 )
|
||||
newVersion="$(semver $latestVersion -p -i patch)"
|
||||
echo "new release on $bamboo_planRepository_branchName with version $newVersion"
|
||||
elif [[ "${bamboo_version_tag}" != "dev" ]]
|
||||
then
|
||||
newVersion="${bamboo_version_tag}"
|
||||
echo "new special version bild with $newVersion"
|
||||
else
|
||||
newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
|
||||
echo "gitTag=${newVersion}" > git.tag
|
||||
echo "dev build with tag ${newVersion}"
|
||||
python3 -m venv build_venv
|
||||
source build_venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
project_name="image-prediction"
|
||||
HOST="nexus.iqser.com"
|
||||
PORT="5001"
|
||||
|
||||
pip install dvc
|
||||
pip install 'dvc[ssh]'
|
||||
dvc pull
|
||||
|
||||
echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
|
||||
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
|
||||
docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
|
||||
docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} .
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "gitTag=${newVersion}" > git.tag
|
||||
mkdir -p ~/.ssh
|
||||
echo "${bamboo_agent_ssh}" | base64 -d >> ~/.ssh/id_rsa
|
||||
|
||||
python3 -m venv build_venv
|
||||
source build_venv/bin/activate
|
||||
python3 -m pip install --upgrade pip
|
||||
pip install poetry
|
||||
|
||||
pip install dvc
|
||||
pip install 'dvc[ssh]'
|
||||
echo "Pulling dvc data"
|
||||
dvc pull
|
||||
|
||||
echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf
|
||||
docker build -f Dockerfile_base -t $SERVICE_NAME_BASE .
|
||||
docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} .
|
||||
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001
|
||||
docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion}
|
||||
# update version in poetry to latest version in git if it is lower
|
||||
check_poetry_version () {
|
||||
currentVersion=$(poetry version -s)
|
||||
tagCount=$(git rev-list --tags --max-count=1 | wc -l)
|
||||
|
||||
if [[ $tagCount -gt 0 ]]
|
||||
then
|
||||
latestVersion=$(git rev-list --tags --max-count=1 | git describe --tags)
|
||||
echo "latest version tag in git: ${latestVersion}"
|
||||
echo "current version in project: ${currentVersion}"
|
||||
|
||||
if [[ "$currentVersion" < "$latestVersion" ]]
|
||||
then
|
||||
echo "project version is behind"
|
||||
echo "setting latest git tag as current version"
|
||||
poetry version "$latestVersion"
|
||||
updateVersion=0
|
||||
elif [[ "$currentVersion" == "$latestVersion" ]]
|
||||
then
|
||||
echo "project version matches"
|
||||
echo "keeping the project version"
|
||||
updateVersion=0
|
||||
else
|
||||
echo "project version is higher"
|
||||
echo "keeping the project version, be aware that a version might have been skipped"
|
||||
updateVersion=1
|
||||
fi
|
||||
else
|
||||
updateVersion=1
|
||||
fi
|
||||
|
||||
return $updateVersion
|
||||
}
|
||||
|
||||
|
||||
if [[ "$bamboo_planRepository_branchName" == "master" ]]
|
||||
then
|
||||
if check_poetry_version
|
||||
then
|
||||
poetry version minor
|
||||
fi
|
||||
newVersion="$(poetry version -s)"
|
||||
echo "new release on master with version: $newVersion"
|
||||
elif [[ "$bamboo_planRepository_branchName" == release* ]]
|
||||
then
|
||||
if check_poetry_version
|
||||
then
|
||||
poetry version patch
|
||||
fi
|
||||
newVersion="$(poetry version -s)"
|
||||
|
||||
echo "new release on $bamboo_planRepository_branchName with version $newVersion"
|
||||
elif [[ "${bamboo_version_tag}" != "dev" ]]
|
||||
then
|
||||
newVersion="${bamboo_version_tag}"
|
||||
echo "new special version build with: $newVersion"
|
||||
else
|
||||
newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
|
||||
echo "gitTag=${newVersion}" > git.tag
|
||||
echo "dev build with tag: ${newVersion}"
|
||||
|
||||
echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@${HOST}/repository/python-combind/simple" >> pip.conf
|
||||
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin ${HOST}:${PORT}
|
||||
docker build \
|
||||
--build-arg SSH_KEY="$(cat ~/.ssh/id_rsa)" -f Dockerfile .
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "gitTag=${newVersion}" > git.tag
|
||||
|
||||
echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@${HOST}/repository/python-combind/simple" >> pip.conf
|
||||
echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin ${HOST}:${PORT}
|
||||
docker build \
|
||||
-f Dockerfile \
|
||||
-t ${HOST}:${PORT}/${project_name}/$SERVICE_NAME:${newVersion} \
|
||||
--build-arg SSH_KEY="$(cat ~/.ssh/id_rsa)"
|
||||
docker push ${HOST}:${PORT}/${project_name}/$SERVICE_NAME:${newVersion}
|
||||
|
||||
@ -5,4 +5,4 @@ mkdir -p ~/.ssh
|
||||
echo "${bamboo_agent_ssh}" | base64 -d >> ~/.ssh/id_rsa
|
||||
echo "host vector.iqser.com" > ~/.ssh/config
|
||||
echo " user bamboo-agent" >> ~/.ssh/config
|
||||
chmod 600 ~/.ssh/config ~/.ssh/id_rsa
|
||||
chmod 600 ~/.ssh/config ~/.ssh/id_rsa
|
||||
|
||||
@ -39,7 +39,7 @@ then
|
||||
-Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \
|
||||
-Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \
|
||||
-Dsonar.python.coverage.reportPaths=reports/coverage.xml
|
||||
|
||||
|
||||
else
|
||||
echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
|
||||
/usr/bin/sonar-scanner/bin/sonar-scanner \
|
||||
|
||||
@ -18,4 +18,4 @@ public class PlanSpecTest {
|
||||
Plan secPlan = new PlanSpec().createSecBuild();
|
||||
EntityPropertiesBuilders.build(secPlan);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1 +1 @@
|
||||
<mxfile host="app.diagrams.net" modified="2022-03-17T15:35:10.371Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36" etag="b-CbBXg6FXQ9T3Px-oLc" version="17.1.1" type="device"><diagram id="tS3WR_Pr6QhNVK3FqSUP" name="Page-1">1ZZRT6QwEMc/DY8mQHdRX93z9JLbmNzGmNxbQ0daLQzpDrL46a/IsCzinneJcd0XaP+dtsN/fkADscg3V06WeokKbBCHahOIb0Ecnydzf22FphPmyXknZM6oTooGYWWegcWQ1cooWI8CCdGSKcdiikUBKY006RzW47B7tONdS5nBRFil0k7VO6NId+rZPBz0azCZ7neOQh7JZR/MwlpLhfWOJC4DsXCI1LXyzQJs613vSzfv+57RbWIOCvqXCZqW9PBref27aZ7xsQ5vTn/cnvAqT9JW/MCwJuNzR8dZU9Nb4bAqFLSrhYG4qLUhWJUybUdrX3uvacqt70W+yeuCI9jsTTja2uDxAcyBXONDeILonWN04hn366EQUR+jd4qQsCa59tl26cEe32CH/sOt+TueoCONGRbS/kQs2YkHIGoYbFkRvuUTqAmFr1zyu2LlUvhLdjG/HtJlQO/VfOq6AyvJPI3z+HAL4wlwpbp/2V0qODxzUTJmLjo4c8nEkxaWFXcLLPzt4ithKI4BQzHBMOc/l8UvAeLrj9/hQTw9NhBnxwDibB+IB+ZvdvZ5/PnucAx6Gds5S4rLPw==</diagram></mxfile>
|
||||
<mxfile host="app.diagrams.net" modified="2022-03-17T15:35:10.371Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36" etag="b-CbBXg6FXQ9T3Px-oLc" version="17.1.1" type="device"><diagram id="tS3WR_Pr6QhNVK3FqSUP" name="Page-1">1ZZRT6QwEMc/DY8mQHdRX93z9JLbmNzGmNxbQ0daLQzpDrL46a/IsCzinneJcd0XaP+dtsN/fkADscg3V06WeokKbBCHahOIb0Ecnydzf22FphPmyXknZM6oTooGYWWegcWQ1cooWI8CCdGSKcdiikUBKY006RzW47B7tONdS5nBRFil0k7VO6NId+rZPBz0azCZ7neOQh7JZR/MwlpLhfWOJC4DsXCI1LXyzQJs613vSzfv+57RbWIOCvqXCZqW9PBref27aZ7xsQ5vTn/cnvAqT9JW/MCwJuNzR8dZU9Nb4bAqFLSrhYG4qLUhWJUybUdrX3uvacqt70W+yeuCI9jsTTja2uDxAcyBXONDeILonWN04hn366EQUR+jd4qQsCa59tl26cEe32CH/sOt+TueoCONGRbS/kQs2YkHIGoYbFkRvuUTqAmFr1zyu2LlUvhLdjG/HtJlQO/VfOq6AyvJPI3z+HAL4wlwpbp/2V0qODxzUTJmLjo4c8nEkxaWFXcLLPzt4ithKI4BQzHBMOc/l8UvAeLrj9/hQTw9NhBnxwDibB+IB+ZvdvZ5/PnucAx6Gds5S4rLPw==</diagram></mxfile>
|
||||
|
||||
@ -1 +0,0 @@
|
||||
Subproject commit f7292c30ad7c7ae5f07cee6925adda096301b60a
|
||||
@ -1 +0,0 @@
|
||||
Subproject commit 64d6a8cec62eeddf26bd71a9aabc28b40dcec901
|
||||
3675
poetry.lock
generated
Normal file
3675
poetry.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
51
pyproject.toml
Normal file
51
pyproject.toml
Normal file
@ -0,0 +1,51 @@
|
||||
[tool.poetry]
|
||||
name = "image-prediction"
|
||||
version = "1.21.1a0"
|
||||
description = "prediction service of the redai-image model"
|
||||
authors = ["Research Team <research@iqser.com>"]
|
||||
license = "all rights reserved"
|
||||
readme = "README.md"
|
||||
packages = [{include = "image_prediction", from = "src"}]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "~3.8"
|
||||
pip-licenses = "^3.5.4"
|
||||
pyinfra = {git = "ssh://git@git.iqser.com:2222/rr/pyinfra.git", branch = "master"}
|
||||
pdf2img = {git = "ssh://git@git.iqser.com:2222/rr/pdf2image.git", branch = "master"}
|
||||
tensorflow = "^2.11.0"
|
||||
pandas = "^1.5.3"
|
||||
frozendict = "^2.3.4"
|
||||
pdfnetpython3 = "^9.4.2"
|
||||
fpdf = "^1.7.2"
|
||||
funcy = "^1.17"
|
||||
requests = "^2.28.2"
|
||||
urllib3 = "^1.26.14"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pylint = "^2.12.1"
|
||||
black = "^21.11b1"
|
||||
pytest = "^6.2.5"
|
||||
pytest-mock = "^3.6.1"
|
||||
ipykernel = "^6.6.0"
|
||||
pre-commit = "^2.16.0"
|
||||
isort = "^5.10.1"
|
||||
Sphinx = "^4.4.0"
|
||||
recommonmark = "^0.7.1"
|
||||
readthedocs-sphinx-ext = "^2.1.4"
|
||||
rinohtype = "^0.5.3"
|
||||
rst2pdf = "^0.99"
|
||||
pydata-sphinx-theme = "^0.8.0"
|
||||
docxbuilder = "^1.2.0"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
norecursedirs = "incl"
|
||||
filterwarnings = ["ignore:.*:DeprecationWarning", ]
|
||||
testpaths = [
|
||||
"tests",
|
||||
]
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
@ -1,5 +0,0 @@
|
||||
[pytest]
|
||||
norecursedirs = incl
|
||||
filterwarnings =
|
||||
ignore:.*:DeprecationWarning
|
||||
ignore:.*:DeprecationWarning
|
||||
@ -1,25 +0,0 @@
|
||||
Flask==2.1.1
|
||||
requests==2.27.1
|
||||
iteration-utilities==0.11.0
|
||||
dvc==2.10.0
|
||||
dvc[ssh]
|
||||
waitress==2.1.1
|
||||
envyaml==1.10.211231
|
||||
dependency-check==0.6.*
|
||||
mlflow==1.24.0
|
||||
numpy==1.22.3
|
||||
tqdm==4.64.0
|
||||
pandas==1.4.2
|
||||
tensorflow==2.8.0
|
||||
PyYAML==6.0
|
||||
pytest~=7.1.0
|
||||
funcy==1.17
|
||||
PyMuPDF==1.19.6
|
||||
fpdf==1.7.2
|
||||
coverage==6.3.2
|
||||
Pillow==9.1.0
|
||||
PDFNetPython3==9.1.0
|
||||
pdf2image==1.16.0
|
||||
frozendict==2.3.0
|
||||
protobuf<=3.20.*
|
||||
prometheus-client==0.13.1
|
||||
@ -13,7 +13,6 @@ def parse_args():
|
||||
|
||||
|
||||
def main(args):
|
||||
|
||||
response = requests.post("http://127.0.0.1:5000", data=open(args.pdf_path, "rb"))
|
||||
response.raise_for_status()
|
||||
predictions = response.json()
|
||||
|
||||
13
setup.py
13
setup.py
@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from distutils.core import setup
|
||||
|
||||
setup(
|
||||
name="image_prediction",
|
||||
version="0.1.0",
|
||||
description="",
|
||||
author="",
|
||||
author_email="",
|
||||
url="",
|
||||
packages=["image_prediction"],
|
||||
)
|
||||
0
data/.gitignore → src/data/.gitignore
vendored
0
data/.gitignore → src/data/.gitignore
vendored
2
src/image_prediction/__init__.py
Normal file
2
src/image_prediction/__init__.py
Normal file
@ -0,0 +1,2 @@
|
||||
from . import *
|
||||
from image_prediction import *
|
||||
@ -14,4 +14,4 @@ DATA_DIR = PACKAGE_ROOT_DIR / "data"
|
||||
|
||||
MLRUNS_DIR = str(DATA_DIR / "mlruns")
|
||||
|
||||
TEST_DATA_DIR = PACKAGE_ROOT_DIR / "test" / "data"
|
||||
TEST_DATA_DIR = "../" / PACKAGE_ROOT_DIR / "tests" / "data"
|
||||
@ -6,7 +6,6 @@ from typing import Iterable
|
||||
from funcy import rcompose, first, compose, second, chunks, identity, rpartial
|
||||
from tqdm import tqdm
|
||||
|
||||
from image_prediction.config import CONFIG
|
||||
from image_prediction.default_objects import (
|
||||
get_formatter,
|
||||
get_mlflow_model_loader,
|
||||
@ -14,19 +13,19 @@ from image_prediction.default_objects import (
|
||||
get_encoder,
|
||||
get_dispatched_extract,
|
||||
)
|
||||
from image_prediction.locations import MLRUNS_DIR
|
||||
from image_prediction.utils.generic import lift, starlift
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
||||
|
||||
|
||||
def load_model(mlruns_dir, config):
|
||||
model_loader = get_mlflow_model_loader(mlruns_dir)
|
||||
model_identifier = config.service.mlflow_run_id
|
||||
return get_image_classifier(model_loader, model_identifier)
|
||||
|
||||
|
||||
def load_pipeline(**kwargs):
|
||||
model_loader = get_mlflow_model_loader(MLRUNS_DIR)
|
||||
model_identifier = CONFIG.service.mlflow_run_id
|
||||
|
||||
pipeline = Pipeline(model_loader, model_identifier, **kwargs)
|
||||
|
||||
return pipeline
|
||||
return Pipeline(**kwargs)
|
||||
|
||||
|
||||
def parallel(*fs):
|
||||
@ -38,16 +37,15 @@ def star(f):
|
||||
|
||||
|
||||
class Pipeline:
|
||||
def __init__(self, model_loader, model_identifier, batch_size=16, verbose=True, **kwargs):
|
||||
def __init__(self, model, batch_size=16, verbose=True, **kwargs):
|
||||
self.verbose = verbose
|
||||
|
||||
extract = get_dispatched_extract(**kwargs)
|
||||
classifier = get_image_classifier(model_loader, model_identifier)
|
||||
reformat = get_formatter()
|
||||
represent = get_encoder()
|
||||
|
||||
split = compose(star(parallel(*map(lift, (first, first, second)))), rpartial(tee, 3))
|
||||
classify = compose(chain.from_iterable, lift(classifier), partial(chunks, batch_size))
|
||||
classify = compose(chain.from_iterable, lift(model), partial(chunks, batch_size))
|
||||
pairwise_apply = compose(star, parallel)
|
||||
join = compose(starlift(lambda prd, rpr, mdt: {"classification": prd, **mdt, "representation": rpr}), star(zip))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user