diff --git a/Dockerfile b/Dockerfile index ce41849..c7d3c41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG BASE_ROOT="nexus.iqser.com:5001/red/" ARG VERSION_TAG=latest -FROM ${BASE_ROOT}detr-server-base:${VERSION_TAG} +FROM ${BASE_ROOT}fb_detr_prediction_container-base:${VERSION_TAG} WORKDIR /app/service diff --git a/bamboo-specs/pom.xml b/bamboo-specs/pom.xml new file mode 100644 index 0000000..40bc09e --- /dev/null +++ b/bamboo-specs/pom.xml @@ -0,0 +1,40 @@ + + 4.0.0 + + + com.atlassian.bamboo + bamboo-specs-parent + 7.1.2 + + + + bamboo-specs + 1.0.0-SNAPSHOT + jar + + + true + + + + + com.atlassian.bamboo + bamboo-specs-api + + + com.atlassian.bamboo + bamboo-specs + + + + + junit + junit + test + + + + + + \ No newline at end of file diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java new file mode 100644 index 0000000..405fc96 --- /dev/null +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -0,0 +1,181 @@ +package buildjob; + +import com.atlassian.bamboo.specs.api.BambooSpec; +import com.atlassian.bamboo.specs.api.builders.BambooKey; +import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; +import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; +import com.atlassian.bamboo.specs.api.builders.permission.Permissions; +import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions; +import com.atlassian.bamboo.specs.api.builders.plan.Job; +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier; +import com.atlassian.bamboo.specs.api.builders.plan.Stage; +import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; +import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; +import com.atlassian.bamboo.specs.api.builders.project.Project; +import com.atlassian.bamboo.specs.builders.task.CheckoutItem; +import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; +import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; +import com.atlassian.bamboo.specs.builders.task.VcsTagTask; +import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; +import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; +import com.atlassian.bamboo.specs.api.builders.Variable; +import com.atlassian.bamboo.specs.util.BambooServer; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; + +/** + * Plan configuration for Bamboo. + * Learn more on: https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs + */ +@BambooSpec +public class PlanSpec { + + private static final String SERVICE_NAME = "fb_detr_prediction_container"; + private static final String SERVICE_NAME_BASE = "fb_detr_prediction_container-base"; + + private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-",""); + + /** + * Run main to publish plan on Bamboo + */ + public static void main(final String[] args) throws Exception { + //By default credentials are read from the '.credentials' file. + BambooServer bambooServer = new BambooServer("http://localhost:8085"); + + Plan plan = new PlanSpec().createDockerBuildPlan(); + bambooServer.publish(plan); + PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); + bambooServer.publish(planPermission); + } + + private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { + Permissions permission = new Permissions() + .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .loggedInUserPermissions(PermissionType.VIEW) + .anonymousUserPermissionView(); + return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); + } + + private Project project() { + return new Project() + .name("RED") + .key(new BambooKey("RED")); + } + + public Plan createDockerBuildPlan() { + return new Plan( + project(), + SERVICE_NAME, new BambooKey(SERVICE_KEY)) + .description("Docker build for fb_detr_prediction_container.") + // .variables() + .stages(new Stage("Build Stage") + .jobs( + new Job("Build Job", new BambooKey("BUILD")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Build Docker container.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh") + .argument(SERVICE_NAME + " " + SERVICE_NAME_BASE)) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock")), + new Job("Sonar Job", new BambooKey("SONAR")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Run Sonarqube scan.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh") + .argument(SERVICE_NAME), + new ScriptTask() + .description("Shut down any running docker containers.") + .location(Location.FILE) + .inlineBody("pip install docker-compose\n" + + "docker-compose down")) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock"))), + new Stage("Licence Stage") + .jobs( + new Job("Git Tag Job", new BambooKey("GITTAG")) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build git tag.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/git-tag.sh"), + new InjectVariablesTask() + .description("Inject git tag.") + .path("git.tag") + .namespace("g") + .scope(InjectVariablesScope.LOCAL), + new VcsTagTask() + .description("${bamboo.g.gitTag}") + .tagName("${bamboo.g.gitTag}") + .defaultRepository()) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:2.7.0")), + new Job("Licence Job", new BambooKey("LICENCE")) + .enabled(false) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build licence.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh")) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) + .linkedRepositories("RED / " + SERVICE_NAME) + .linkedRepositories("RR / redai_image") + .triggers(new BitbucketServerTrigger()) + .planBranchManagement(new PlanBranchManagement() + .createForVcsBranch() + .delete(new BranchCleanup() + .whenInactiveInRepositoryAfterDays(14)) + .notificationForCommitters()); + } + + +} diff --git a/bamboo-specs/src/main/resources/scripts/create-licence.sh b/bamboo-specs/src/main/resources/scripts/create-licence.sh new file mode 100755 index 0000000..a9054cd --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/create-licence.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +if [[ \"${bamboo_version_tag}\" != \"dev\" ]] +then + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + versions:set \ + -DnewVersion=${bamboo_version_tag} + + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + -B clean deploy \ + -e -DdeployAtEnd=true \ + -Dmaven.wagon.http.ssl.insecure=true \ + -Dmaven.wagon.http.ssl.allowall=true \ + -Dmaven.wagon.http.ssl.ignore.validity.dates=true \ + -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/docker-build.sh b/bamboo-specs/src/main/resources/scripts/docker-build.sh new file mode 100755 index 0000000..f17638f --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/docker-build.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +SERVICE_NAME=$1 +SERVICE_NAME_BASE=$2 + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +pip install dvc +pip install 'dvc[ssh]' +dvc pull + +echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf +docker build -f Dockerfile_base -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} . +docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} . +echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 +docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} diff --git a/bamboo-specs/src/main/resources/scripts/git-tag.sh b/bamboo-specs/src/main/resources/scripts/git-tag.sh new file mode 100755 index 0000000..2005666 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/git-tag.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +if [[ "${bamboo_version_tag}" = "dev" ]] +then + echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag +else + echo "gitTag=${bamboo_version_tag}" > git.tag +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh new file mode 100755 index 0000000..ca8c1f5 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +export JAVA_HOME=/usr/bin/sonar-scanner/jre + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +echo "dev setup for unit test and coverage" + +pip install -e incl/detr/ + +pip install -r requirements.txt + + +echo "DVC pull step" +dvc pull + +#echo "coverage calculation" +#coverage run -m pytest -s --disable-warnings incl/image_service +#echo "coverage report generation" +#coverage report -m +#coverage xml + +SERVICE_NAME=$1 + +echo "dependency-check:aggregate" +mkdir -p reports +dependency-check --enableExperimental -f JSON -f HTML -f XML \ + --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ + --exclude "build_venv/**" --exclude "**/__pycache__/**" + +if [[ -z "${bamboo_repository_pr_key}" ]] +then + echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" + /usr/bin/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml + +else + echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" + /usr/bin/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=incl/image_service/image_service,incl/redai_image/redai/redai/backend,incl/redai_image/redai/redai/utils,src,incl/redai_image/redai/redai/model/efficientnetb0mod.py \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ + -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ + -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml +fi diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java new file mode 100644 index 0000000..fada379 --- /dev/null +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -0,0 +1,16 @@ +package buildjob; + + +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException; +import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; +import org.junit.Test; + +public class PlanSpecTest { + @Test + public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createDockerBuildPlan(); + + EntityPropertiesBuilders.build(plan); + } +} \ No newline at end of file diff --git a/src/run_service.py b/src/run_service.py index 17f7492..81ac972 100644 --- a/src/run_service.py +++ b/src/run_service.py @@ -57,6 +57,18 @@ def run_server(resume): app = Flask(__name__) + @app.route("/ready", methods=["GET"]) + def ready(): + resp = jsonify("OK") + resp.status_code = 200 + return resp + + @app.route("/health", methods=["GET"]) + def healthy(): + resp = jsonify("OK") + resp.status_code = 200 + return resp + @app.route("/", methods=["POST"]) def predict_request():