diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 8ff9112..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "incl/pyinfra"] - path = incl/pyinfra - url = ssh://git@git.iqser.com:2222/rr/pyinfra.git diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bamboo-specs/pom.xml b/bamboo-specs/pom.xml deleted file mode 100644 index 40bc09e..0000000 --- a/bamboo-specs/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - 4.0.0 - - - com.atlassian.bamboo - bamboo-specs-parent - 7.1.2 - - - - bamboo-specs - 1.0.0-SNAPSHOT - jar - - - true - - - - - com.atlassian.bamboo - bamboo-specs-api - - - com.atlassian.bamboo - bamboo-specs - - - - - junit - junit - test - - - - - - \ No newline at end of file diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java deleted file mode 100644 index 5ed8768..0000000 --- a/bamboo-specs/src/main/java/buildjob/PlanSpec.java +++ /dev/null @@ -1,180 +0,0 @@ -package buildjob; - -import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask; - -import java.time.LocalTime; - -import com.atlassian.bamboo.specs.api.BambooSpec; -import com.atlassian.bamboo.specs.api.builders.BambooKey; -import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; -import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; -import com.atlassian.bamboo.specs.api.builders.permission.Permissions; -import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions; -import com.atlassian.bamboo.specs.api.builders.plan.Job; -import com.atlassian.bamboo.specs.api.builders.plan.Plan; -import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier; -import com.atlassian.bamboo.specs.api.builders.plan.Stage; -import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; -import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; -import com.atlassian.bamboo.specs.api.builders.project.Project; -import com.atlassian.bamboo.specs.builders.task.CheckoutItem; -import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; -import com.atlassian.bamboo.specs.builders.task.ScriptTask; -import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; -import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; -import com.atlassian.bamboo.specs.builders.task.VcsTagTask; -import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; -import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger; -import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; -import com.atlassian.bamboo.specs.api.builders.Variable; -import com.atlassian.bamboo.specs.util.BambooServer; -import com.atlassian.bamboo.specs.builders.task.ScriptTask; -import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; - -/** - * Plan configuration for Bamboo. - * Learn more on: https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs - */ -@BambooSpec -public class PlanSpec { - - private static final String SERVICE_NAME = "image-prediction"; - private static final String SERVICE_NAME_BASE = "image-prediction-base"; - - private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-","").replaceAll("_",""); - - /** - * Run main to publish plan on Bamboo - */ - public static void main(final String[] args) throws Exception { - //By default credentials are read from the '.credentials' file. - BambooServer bambooServer = new BambooServer("http://localhost:8085"); - - Plan plan = new PlanSpec().createDockerBuildPlan(); - bambooServer.publish(plan); - PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); - bambooServer.publish(planPermission); - - Plan secPlan = new PlanSpec().createSecBuild(); - bambooServer.publish(secPlan); - PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier()); - bambooServer.publish(secPlanPermission); - } - - private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { - Permissions permission = new Permissions() - .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) - .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) - .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) - .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) - .loggedInUserPermissions(PermissionType.VIEW) - .anonymousUserPermissionView(); - return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); - } - - private Project project() { - return new Project() - .name("RED") - .key(new BambooKey("RED")); - } - - public Plan createDockerBuildPlan() { - return new Plan( - project(), - SERVICE_NAME, new BambooKey(SERVICE_KEY)) - .description("Docker build for image-prediction.") - .stages( - new Stage("Build Stage") - .jobs( - new Job("Build Job", new BambooKey("BUILD")) - .tasks( - new CleanWorkingDirectoryTask() - .description("Clean working directory.") - .enabled(true), - new VcsCheckoutTask() - .description("Checkout default repository.") - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Set config and keys.") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"), - new ScriptTask() - .description("Build Docker container.") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh") - .argument(SERVICE_NAME + " " + SERVICE_NAME_BASE), - new InjectVariablesTask() - .description("Inject git tag.") - .path("git.tag") - .namespace("g") - .scope(InjectVariablesScope.LOCAL), - new VcsTagTask() - .description("${bamboo.g.gitTag}") - .tagName("${bamboo.g.gitTag}") - .defaultRepository()) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/release_build:4.5.0") - .volume("/var/run/docker.sock", "/var/run/docker.sock")), - new Job("Licence Job", new BambooKey("LICENCE")) - .enabled(false) - .tasks( - new VcsCheckoutTask() - .description("Checkout default repository.") - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Build licence.") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh")) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") - .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") - .volume("/var/run/docker.sock", "/var/run/docker.sock")))) - .linkedRepositories("RR / " + SERVICE_NAME) - .linkedRepositories("RR / redai_image") - .triggers( - new BitbucketServerTrigger()) - .planBranchManagement( - new PlanBranchManagement() - .createForVcsBranch() - .delete( - new BranchCleanup() - .whenInactiveInRepositoryAfterDays(14)) - .notificationForCommitters()); - } - - public Plan createSecBuild() { - return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC")).description("Security Analysis Plan") - .stages(new Stage("Default Stage").jobs( - new Job("Sonar Job", new BambooKey("SONAR")) - .tasks( - new CleanWorkingDirectoryTask() - .description("Clean working directory.") - .enabled(true), - new VcsCheckoutTask() - .description("Checkout default repository.") - .checkoutItems(new CheckoutItem().defaultRepository()), - new ScriptTask() - .description("Set config and keys.") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/key-prepare.sh"), - new ScriptTask() - .description("Run Sonarqube scan.") - .location(Location.FILE) - .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh") - .argument(SERVICE_NAME)) - .dockerConfiguration( - new DockerConfiguration() - .image("nexus.iqser.com:5001/infra/release_build:4.2.0") - .volume("/var/run/docker.sock", "/var/run/docker.sock")))) - .linkedRepositories("RR / " + SERVICE_NAME) - .triggers( - new ScheduledTrigger() - .scheduleOnceDaily(LocalTime.of(23, 00))) - .planBranchManagement( - new PlanBranchManagement() - .createForVcsBranchMatching("release.*") - .notificationForCommitters()); - } -} diff --git a/bamboo-specs/src/main/resources/scripts/create-licence.sh b/bamboo-specs/src/main/resources/scripts/create-licence.sh deleted file mode 100755 index a9054cd..0000000 --- a/bamboo-specs/src/main/resources/scripts/create-licence.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -if [[ \"${bamboo_version_tag}\" != \"dev\" ]] -then - ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ - -f ${bamboo_build_working_directory}/pom.xml \ - versions:set \ - -DnewVersion=${bamboo_version_tag} - - ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ - -f ${bamboo_build_working_directory}/pom.xml \ - -B clean deploy \ - -e -DdeployAtEnd=true \ - -Dmaven.wagon.http.ssl.insecure=true \ - -Dmaven.wagon.http.ssl.allowall=true \ - -Dmaven.wagon.http.ssl.ignore.validity.dates=true \ - -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases -fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/docker-build.sh b/bamboo-specs/src/main/resources/scripts/docker-build.sh deleted file mode 100755 index 5bc8302..0000000 --- a/bamboo-specs/src/main/resources/scripts/docker-build.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -e - -SERVICE_NAME=$1 -SERVICE_NAME_BASE=$2 -if [[ "$bamboo_planRepository_branchName" == "master" ]] -then - branchVersion=$(cat version.yaml | grep -Eo "version: .*" | sed -s 's|version: \(.*\)\..*\..*|\1|g') - latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 ) - newVersion="$(semver $latestVersion -p -i minor)" - echo "new release on master with version $newVersion" -elif [[ "$bamboo_planRepository_branchName" == release* ]] -then - branchVersion=$(echo $bamboo_planRepository_branchName | sed -s 's|release\/\([0-9]\+\.[0-9]\+\)\.x|\1|') - latestVersion=$( semver $(git tag -l "${branchVersion}.*" ) | tail -n1 ) - newVersion="$(semver $latestVersion -p -i patch)" - echo "new release on $bamboo_planRepository_branchName with version $newVersion" -elif [[ "${bamboo_version_tag}" != "dev" ]] -then - newVersion="${bamboo_version_tag}" - echo "new special version bild with $newVersion" -else - newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" - echo "gitTag=${newVersion}" > git.tag - dev_tag="dev" - echo "dev build with tag $dev_tag" - python3 -m venv build_venv - source build_venv/bin/activate - python3 -m pip install --upgrade pip - - pip install dvc - pip install 'dvc[ssh]' - dvc pull - - echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf - echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 - docker build -f Dockerfile_base -t $SERVICE_NAME_BASE . - docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:$dev_tag . - docker push nexus.iqser.com:5001/red/$SERVICE_NAME:$dev_tag - exit 0 -fi - -echo "gitTag=${newVersion}" > git.tag - -python3 -m venv build_venv -source build_venv/bin/activate -python3 -m pip install --upgrade pip - -pip install dvc -pip install 'dvc[ssh]' -echo "Pulling dvc data" -dvc pull - -echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf -docker build -f Dockerfile_base -t $SERVICE_NAME_BASE . -docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} . -echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 -docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${newVersion} \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/key-prepare.sh b/bamboo-specs/src/main/resources/scripts/key-prepare.sh deleted file mode 100755 index bf71668..0000000 --- a/bamboo-specs/src/main/resources/scripts/key-prepare.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e - -mkdir -p ~/.ssh -echo "${bamboo_agent_ssh}" | base64 -d >> ~/.ssh/id_rsa -echo "host vector.iqser.com" > ~/.ssh/config -echo " user bamboo-agent" >> ~/.ssh/config -chmod 600 ~/.ssh/config ~/.ssh/id_rsa \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh deleted file mode 100755 index 9286748..0000000 --- a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -set -e - -export JAVA_HOME=/usr/bin/sonar-scanner/jre - -python3 -m venv build_venv -source build_venv/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip install dependency-check -python3 -m pip install coverage - -echo "coverage report generation" - -bash run_tests.sh - -if [ ! -f reports/coverage.xml ] -then - exit 1 -fi - -SERVICE_NAME=$1 - -echo "dependency-check:aggregate" -mkdir -p reports -dependency-check --enableExperimental -f JSON -f HTML -f XML \ - --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ - --exclude "build_venv/**" --exclude "**/__pycache__/**" --exclude "bamboo-specs/**" - -if [[ -z "${bamboo_repository_pr_key}" ]] -then - echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" - /usr/bin/sonar-scanner/bin/sonar-scanner \ - -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.sources=image_prediction \ - -Dsonar.host.url=https://sonarqube.iqser.com \ - -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ - -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ - -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ - -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ - -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ - -Dsonar.python.coverage.reportPaths=reports/coverage.xml - -else - echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" - /usr/bin/sonar-scanner/bin/sonar-scanner \ - -Dsonar.projectKey=RED_$SERVICE_NAME \ - -Dsonar.sources=image_prediction \ - -Dsonar.host.url=https://sonarqube.iqser.com \ - -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ - -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ - -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ - -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \ - -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ - -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ - -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ - -Dsonar.python.coverage.reportPaths=reports/coverage.xml -fi diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java deleted file mode 100644 index 20128c1..0000000 --- a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package buildjob; - - -import com.atlassian.bamboo.specs.api.builders.plan.Plan; -import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException; -import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; -import org.junit.Test; - -public class PlanSpecTest { - @Test - public void checkYourPlanOffline() throws PropertiesValidationException { - Plan plan = new PlanSpec().createDockerBuildPlan(); - EntityPropertiesBuilders.build(plan); - } - - @Test - public void checkYourSecPlanOffline() throws PropertiesValidationException { - Plan secPlan = new PlanSpec().createSecBuild(); - EntityPropertiesBuilders.build(secPlan); - } -} \ No newline at end of file diff --git a/incl/__init__.py b/incl/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/incl/pyinfra b/incl/pyinfra deleted file mode 160000 index d838413..0000000 --- a/incl/pyinfra +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d8384135005b94290f9b79c3a72fb3adb025cfbe diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3559e63..0000000 --- a/requirements.txt +++ /dev/null @@ -1,27 +0,0 @@ -Flask==2.1.1 -requests==2.27.1 -iteration-utilities==0.11.0 -dvc==2.10.0 -dvc[ssh] -waitress==2.1.1 -envyaml==1.10.211231 -dependency-check==0.6.* -mlflow==1.24.0 -numpy==1.22.3 -tqdm==4.64.0 -pandas==1.4.2 -tensorflow==2.8.0 -PyYAML==6.0 -pytest~=7.1.0 -funcy==1.17 -PyMuPDF==1.19.6 -fpdf==1.7.2 -coverage==6.3.2 -Pillow==9.1.0 -PDFNetPython3==9.1.0 -pdf2image==1.16.0 -frozendict==2.3.0 -protobuf<=3.20.* -prometheus-client==0.13.1 -fsspec==2022.11.0 -PyMonad==2.4.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 53742f4..0000000 --- a/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -from distutils.core import setup - -setup( - name="image_prediction", - version="0.1.0", - description="", - author="", - author_email="", - url="", - packages=["image_prediction"], -) diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 4eb136e..0000000 --- a/sonar-project.properties +++ /dev/null @@ -1,4 +0,0 @@ -sonar.exclusions=bamboo-specs/**, **/test_data/** -sonar.c.file.suffixes=- -sonar.cpp.file.suffixes=- -sonar.objc.file.suffixes=- diff --git a/version.yaml b/version.yaml deleted file mode 100644 index f2c2fb5..0000000 --- a/version.yaml +++ /dev/null @@ -1,2 +0,0 @@ -name: image-prediction-v1 -version: 1.x.x \ No newline at end of file