diff --git a/.gitignore b/.gitignore index 2c82e04..82a55cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .venv __pycache__ data/ +build_venv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..92c4861 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.8 as builder + +# Use a virtual environment. +RUN python -m venv /app/venv +ENV PATH="/app/venv/bin:$PATH" + +# Upgrade pip. +RUN python -m pip install --upgrade pip + +# Make a directory for the service files and copy the service repo into the container. +WORKDIR /app/service +COPY . ./ + +RUN python3 -m pip install -e . +RUN python3 -m pip install -r requirements.txt + +# Run the service loop. +CMD ["python", "mini_queue/run.py"] diff --git a/bamboo-specs/pom.xml b/bamboo-specs/pom.xml new file mode 100644 index 0000000..40bc09e --- /dev/null +++ b/bamboo-specs/pom.xml @@ -0,0 +1,40 @@ + + 4.0.0 + + + com.atlassian.bamboo + bamboo-specs-parent + 7.1.2 + + + + bamboo-specs + 1.0.0-SNAPSHOT + jar + + + true + + + + + com.atlassian.bamboo + bamboo-specs-api + + + com.atlassian.bamboo + bamboo-specs + + + + + junit + junit + test + + + + + + \ No newline at end of file diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java new file mode 100644 index 0000000..e64d541 --- /dev/null +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -0,0 +1,179 @@ +package buildjob; + +import com.atlassian.bamboo.specs.api.BambooSpec; +import com.atlassian.bamboo.specs.api.builders.BambooKey; +import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; +import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; +import com.atlassian.bamboo.specs.api.builders.permission.Permissions; +import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions; +import com.atlassian.bamboo.specs.api.builders.plan.Job; +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier; +import com.atlassian.bamboo.specs.api.builders.plan.Stage; +import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; +import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; +import com.atlassian.bamboo.specs.api.builders.project.Project; +import com.atlassian.bamboo.specs.builders.task.CheckoutItem; +import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; +import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; +import com.atlassian.bamboo.specs.builders.task.VcsTagTask; +import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; +import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; +import com.atlassian.bamboo.specs.api.builders.Variable; +import com.atlassian.bamboo.specs.util.BambooServer; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; + +/** + * Plan configuration for Bamboo. + * Learn more on: https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs + */ +@BambooSpec +public class PlanSpec { + + private static final String SERVICE_NAME = "mini-queue-service-v1"; + + private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-",""); + + /** + * Run main to publish plan on Bamboo + */ + public static void main(final String[] args) throws Exception { + //By default credentials are read from the '.credentials' file. + BambooServer bambooServer = new BambooServer("http://localhost:8085"); + + Plan plan = new PlanSpec().createDockerBuildPlan(); + bambooServer.publish(plan); + PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); + bambooServer.publish(planPermission); + } + + private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { + Permissions permission = new Permissions() + .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .loggedInUserPermissions(PermissionType.VIEW) + .anonymousUserPermissionView(); + return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); + } + + private Project project() { + return new Project() + .name("RED") + .key(new BambooKey("RED")); + } + + public Plan createDockerBuildPlan() { + return new Plan( + project(), + SERVICE_NAME, new BambooKey(SERVICE_KEY)) + .description("Docker build for mini-queue-service-v1") + // .variables() + .stages(new Stage("Build Stage") + .jobs( + new Job("Build Job", new BambooKey("BUILD")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Build Docker container.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh") + .argument(SERVICE_NAME) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock")), + new Job("Sonar Job", new BambooKey("SONAR")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Run Sonarqube scan.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh") + .argument(SERVICE_NAME), + new ScriptTask() + .description("Shut down any running docker containers.") + .location(Location.FILE) + .inlineBody("pip install docker-compose\n" + + "docker-compose down")) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock"))), + new Stage("Licence Stage") + .jobs( + new Job("Git Tag Job", new BambooKey("GITTAG")) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build git tag.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/git-tag.sh"), + new InjectVariablesTask() + .description("Inject git tag.") + .path("git.tag") + .namespace("g") + .scope(InjectVariablesScope.LOCAL), + new VcsTagTask() + .description("${bamboo.g.gitTag}") + .tagName("${bamboo.g.gitTag}") + .defaultRepository()) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:2.7.0")), + new Job("Licence Job", new BambooKey("LICENCE")) + .enabled(false) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build licence.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh")) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) + .linkedRepositories("RED / " + SERVICE_NAME) + .triggers(new BitbucketServerTrigger()) + .planBranchManagement(new PlanBranchManagement() + .createForVcsBranch() + .delete(new BranchCleanup() + .whenInactiveInRepositoryAfterDays(14)) + .notificationForCommitters()); + } + + +} diff --git a/bamboo-specs/src/main/resources/scripts/create-licence.sh b/bamboo-specs/src/main/resources/scripts/create-licence.sh new file mode 100755 index 0000000..a9054cd --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/create-licence.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +if [[ \"${bamboo_version_tag}\" != \"dev\" ]] +then + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + versions:set \ + -DnewVersion=${bamboo_version_tag} + + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + -B clean deploy \ + -e -DdeployAtEnd=true \ + -Dmaven.wagon.http.ssl.insecure=true \ + -Dmaven.wagon.http.ssl.allowall=true \ + -Dmaven.wagon.http.ssl.ignore.validity.dates=true \ + -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/docker-build.sh b/bamboo-specs/src/main/resources/scripts/docker-build.sh new file mode 100755 index 0000000..93019b0 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/docker-build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +SERVICE_NAME=$1 + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf +docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} . +echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 +docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} + diff --git a/bamboo-specs/src/main/resources/scripts/git-tag.sh b/bamboo-specs/src/main/resources/scripts/git-tag.sh new file mode 100755 index 0000000..2005666 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/git-tag.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +if [[ "${bamboo_version_tag}" = "dev" ]] +then + echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag +else + echo "gitTag=${bamboo_version_tag}" > git.tag +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh new file mode 100755 index 0000000..96afd2a --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +export JAVA_HOME=/usr/bin/sonar-scanner/jre + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +SERVICE_NAME=$1 + +echo "dependency-check:aggregate" +mkdir -p reports +dependency-check --enableExperimental -f JSON -f XML \ + --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ + --exclude "build_venv/**" --exclude "**/__pycache__/**" + +if [[ -z "${bamboo_repository_pr_key}" ]] +then + echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" + /usr/bin/sonar-scanner/bin/sonar-scanner -X\ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml + +else + echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" + /usr/bin/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ + -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ + -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml +fi diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java new file mode 100644 index 0000000..fada379 --- /dev/null +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -0,0 +1,16 @@ +package buildjob; + + +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException; +import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; +import org.junit.Test; + +public class PlanSpecTest { + @Test + public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createDockerBuildPlan(); + + EntityPropertiesBuilders.build(plan); + } +} \ No newline at end of file diff --git a/mini_queue/consumer.py b/mini_queue/consumer.py index c4c4f73..89990b0 100644 --- a/mini_queue/consumer.py +++ b/mini_queue/consumer.py @@ -1,8 +1,8 @@ -from functools import partial import pika from retry import retry -from mini_queue.utils.config import CONFIG + from mini_queue.producer import produce_response +from mini_queue.utils.config import CONFIG def callback(ch, method, properties, body): @@ -15,7 +15,10 @@ def callback(ch, method, properties, body): def init_params(): credentials = pika.PlainCredentials(CONFIG.rabbitmq.user, CONFIG.rabbitmq.password) parameters = pika.ConnectionParameters( - host=CONFIG.rabbitmq.host, port=CONFIG.rabbitmq.port, heartbeat=CONFIG.rabbitmq.heartbeat, credentials=credentials + host=CONFIG.rabbitmq.host, + port=CONFIG.rabbitmq.port, + heartbeat=CONFIG.rabbitmq.heartbeat, + credentials=credentials, ) return parameters @@ -24,15 +27,14 @@ def init_params(): def consume(parameters, queue): connection = pika.BlockingConnection(parameters) channel = connection.channel() - while True: - try: - channel.basic_consume(queue=queue, auto_ack=True, on_message_callback=callback) - print(" [*] Waiting for messages. To exit press CTRL+C") + try: + channel.basic_consume(queue=queue, auto_ack=True, on_message_callback=callback) + print(" [*] Waiting for messages. To exit press CTRL+C") - channel.start_consuming() - except pika.exceptions.ConnectionClosedByBroker: - pass - except pika.exceptions.AMQPChannelError: - pass - except pika.exceptions.AMQPConnectionError: - pass + channel.start_consuming() + except pika.exceptions.ConnectionClosedByBroker: + pass + except pika.exceptions.AMQPChannelError: + pass + except pika.exceptions.AMQPConnectionError: + pass diff --git a/mini_queue/producer.py b/mini_queue/producer.py index fc44545..8d829d6 100644 --- a/mini_queue/producer.py +++ b/mini_queue/producer.py @@ -1,7 +1,5 @@ import pika -from mini_queue.utils.config import CONFIG - def produce_response(parameters, queue, body): connection = pika.BlockingConnection(parameters) diff --git a/mini_queue/run.py b/mini_queue/run.py index 2115452..a262f63 100644 --- a/mini_queue/run.py +++ b/mini_queue/run.py @@ -4,6 +4,7 @@ from mini_queue.utils.config import CONFIG if __name__ == "__main__": + print("startet happy pikachu!") queue = CONFIG.rabbitmq.queues.input parameters = init_params() consume(parameters, queue) diff --git a/requirements.txt b/requirements.txt index ecba2e8..8cb2919 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pika retry +envyaml \ No newline at end of file diff --git a/scripts/mock_publish.py b/scripts/mock_publish.py index 22fae71..542ab65 100644 --- a/scripts/mock_publish.py +++ b/scripts/mock_publish.py @@ -1,3 +1,5 @@ +import json + import pika from mini_queue.producer import produce_response @@ -9,7 +11,10 @@ if __name__ == "__main__": credentials = pika.PlainCredentials(CONFIG.rabbitmq.user, CONFIG.rabbitmq.password) queue = CONFIG.rabbitmq.queues.input parameters = pika.ConnectionParameters( - host=CONFIG.rabbitmq.host, port=CONFIG.rabbitmq.port, heartbeat=CONFIG.rabbitmq.heartbeat, credentials=credentials + host=CONFIG.rabbitmq.host, + port=CONFIG.rabbitmq.port, + heartbeat=CONFIG.rabbitmq.heartbeat, + credentials=credentials, ) - body = "Pika pika!" + body = json.dumps({"fileId": "234", "dossierId": "3403"}) produce_response(parameters, queue, body)