Merge branch 'RES-109-add-gitlab-ci' into 'master'

RES-109: add gitlab ci

See merge request knecon/research/pyinfra!71
This commit is contained in:
Christoph Schabert 2023-04-20 09:43:36 +02:00
commit e67ebc27b1
11 changed files with 60 additions and 487 deletions

59
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,59 @@
default:
image: python:3.8
before_script:
- echo "Run started by ${GITLAB_USER_NAME}"
- echo "Pipeline on branch ${CI_COMMIT_REF_SLUG}"
- echo "$(pip cache dir)"
# install poetry
- pip install poetry
# poetry config
- poetry config installer.max-workers 10
- poetry config virtualenvs.in-project true
- poetry config repositories.gitlab https://gitlab.knecon.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi
# install package & dependencies
- poetry install --with=dev
# activate virtual environment
- source .venv/bin/activate
variables:
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
cache:
key: "${CI_JOB_NAME}"
paths:
- .venv/
- ${PIP_CACHE_DIR}/.cache/pip
stages:
- test
- build
# run-tests:
# stage: test
# script:
# - echo "Running pytest against the package"
# - pytest ./tests
build-pkg-dev:
stage: build
script:
- export PKG_VERSION="$(poetry version -s)-dev"
- echo "Building DEV package with version ${PKG_VERSION}"
- poetry version ${PKG_VERSION}
- poetry build
- echo "Publishing ..."
- poetry publish --repository gitlab -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD}
when: manual
build-pkg-prod:
stage: build
script:
- export PKG_VERSION="$(poetry version -s)"
- echo "Building PROD package with version ${PKG_VERSION}"
- poetry version ${PKG_VERSION}
- poetry build
- echo "Publishing ..."
- poetry publish --repository gitlab -u ${CI_REGISTRY_USER} -p ${CI_REGISTRY_PASSWORD}
rules:
- if: $CI_COMMIT_TAG

View File

@ -1,40 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.atlassian.bamboo</groupId>
<artifactId>bamboo-specs-parent</artifactId>
<version>7.1.2</version>
<relativePath/>
</parent>
<artifactId>bamboo-specs</artifactId>
<version>1.0.0-SNAPSHOT</version>
<packaging>jar</packaging>
<properties>
<sonar.skip>true</sonar.skip>
</properties>
<dependencies>
<dependency>
<groupId>com.atlassian.bamboo</groupId>
<artifactId>bamboo-specs-api</artifactId>
</dependency>
<dependency>
<groupId>com.atlassian.bamboo</groupId>
<artifactId>bamboo-specs</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<!-- run 'mvn test' to perform offline validation of the plan -->
<!-- run 'mvn -Ppublish-specs' to upload the plan to your Bamboo server -->
</project>

View File

@ -1,213 +0,0 @@
package buildjob;
import static com.atlassian.bamboo.specs.builders.task.TestParserTask.createJUnitParserTask;
import java.time.LocalTime;
import com.atlassian.bamboo.specs.api.BambooSpec;
import com.atlassian.bamboo.specs.api.builders.BambooKey;
import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration;
import com.atlassian.bamboo.specs.api.builders.permission.PermissionType;
import com.atlassian.bamboo.specs.api.builders.permission.Permissions;
import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions;
import com.atlassian.bamboo.specs.api.builders.plan.Job;
import com.atlassian.bamboo.specs.api.builders.plan.Plan;
import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier;
import com.atlassian.bamboo.specs.api.builders.plan.Stage;
import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup;
import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement;
import com.atlassian.bamboo.specs.api.builders.project.Project;
import com.atlassian.bamboo.specs.builders.task.CheckoutItem;
import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask;
import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask;
import com.atlassian.bamboo.specs.builders.task.VcsTagTask;
import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger;
import com.atlassian.bamboo.specs.builders.trigger.ScheduledTrigger;
import com.atlassian.bamboo.specs.model.task.InjectVariablesScope;
import com.atlassian.bamboo.specs.api.builders.Variable;
import com.atlassian.bamboo.specs.util.BambooServer;
import com.atlassian.bamboo.specs.builders.task.ScriptTask;
import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location;
/**
* Plan configuration for Bamboo.
* Learn more on: <a href=
* "https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs">https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs</a>
*/
@BambooSpec
public class PlanSpec {
private static final String REPOSITORY_KEY = "RR";
// this is the repo name
private static final String SERVICE_NAME = "pyinfra";
private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-", "").replaceAll("_", "");
private static final String PROJECT_NAME = "RED";
private static final String PROJECT_KEY = "RED";
private static final String HOST = "nexus.iqser.com";
private static final String PORT = "5001";;
private static final String INFRA_URI = HOST + ":" + PORT + "/infra/release_build:4.5.0";
private static final String MAVEN_URI = HOST + ":" + PORT + "/infra/maven:3.8.4-openjdk-17-slim";
/**
* Run main to publish plan on Bamboo
*/
public static void main(final String[] args) throws Exception {
// By default credentials are read from the '.credentials' file.
BambooServer bambooServer = new BambooServer("http://localhost:8085");
Plan plan = new PlanSpec().createBuildPlan();
bambooServer.publish(plan);
PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier());
bambooServer.publish(planPermission);
Plan secPlan = new PlanSpec().createSecBuild();
bambooServer.publish(secPlan);
PlanPermissions secPlanPermission = new PlanSpec().createPlanPermission(secPlan.getIdentifier());
bambooServer.publish(secPlanPermission);
}
private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) {
Permissions permission = new Permissions()
.userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW,
PermissionType.ADMIN,
PermissionType.CLONE, PermissionType.BUILD)
.groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW,
PermissionType.CLONE,
PermissionType.BUILD)
.groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW,
PermissionType.CLONE,
PermissionType.BUILD)
.groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE,
PermissionType.BUILD)
.loggedInUserPermissions(PermissionType.VIEW)
.anonymousUserPermissionView();
return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey())
.permissions(permission);
}
private Project project() {
return new Project()
.name(PROJECT_NAME)
.key(new BambooKey(PROJECT_KEY));
}
public Plan createBuildPlan() {
return new Plan(
project(),
SERVICE_NAME, new BambooKey(SERVICE_KEY))
.description("Build for " + SERVICE_NAME)
// .variables()
.stages(
new Stage("Build Stage")
.jobs(
new Job("Build Job", new BambooKey("BUILD"))
.tasks(
new CleanWorkingDirectoryTask()
.description("Clean working directory.")
.enabled(true),
new VcsCheckoutTask()
.description("Checkout default repository.")
.checkoutItems(new CheckoutItem()
.defaultRepository()),
// new ScriptTask()
// .description("Set config and keys.")
// .location(Location.FILE)
// .fileFromPath(
// "bamboo-specs/src/main/resources/scripts/config-keys.sh"),
new ScriptTask()
.description("Tag Version.")
.location(Location.FILE)
.fileFromPath(
"bamboo-specs/src/main/resources/scripts/git-tag.sh")
.argument(SERVICE_NAME),
new InjectVariablesTask()
.description("Inject git tag.")
.path("git.tag")
.namespace("g")
.scope(InjectVariablesScope.LOCAL),
new VcsTagTask()
.description("${bamboo.g.gitTag}")
.tagName("${bamboo.g.gitTag}")
.defaultRepository())
.dockerConfiguration(
new DockerConfiguration()
.image(INFRA_URI)
.volume("/var/run/docker.sock",
"/var/run/docker.sock"))),
new Stage("License Stage")
.jobs(
new Job("License Job", new BambooKey("LICENSE"))
.enabled(true)
.tasks(
new VcsCheckoutTask()
.description("Checkout default repository.")
.checkoutItems(new CheckoutItem()
.defaultRepository()),
new ScriptTask()
.description("Build licence.")
.location(Location.FILE)
.fileFromPath(
"bamboo-specs/src/main/resources/scripts/create-licence.sh"))
.dockerConfiguration(
new DockerConfiguration()
.image(MAVEN_URI)
.volume("/etc/maven/settings.xml",
"/usr/share/maven/ref/settings.xml")
.volume("/var/run/docker.sock",
"/var/run/docker.sock"))))
.linkedRepositories(REPOSITORY_KEY + " / " + SERVICE_NAME)
.triggers(
new BitbucketServerTrigger())
.planBranchManagement(
new PlanBranchManagement()
.createForVcsBranch()
.delete(
new BranchCleanup()
.whenInactiveInRepositoryAfterDays(
14))
.notificationForCommitters());
}
public Plan createSecBuild() {
return new Plan(project(), SERVICE_NAME + "-Sec", new BambooKey(SERVICE_KEY + "SEC"))
.description("Security Analysis Plan")
.stages(new Stage("Default Stage").jobs(
new Job("Sonar Job", new BambooKey("SONAR"))
.enabled(false)
.tasks(
new CleanWorkingDirectoryTask()
.description("Clean working directory.")
.enabled(true),
new VcsCheckoutTask()
.description("Checkout default repository.")
.checkoutItems(new CheckoutItem()
.defaultRepository()),
new ScriptTask()
.description("Set config and keys.")
.location(Location.FILE)
.fileFromPath("bamboo-specs/src/main/resources/scripts/config-keys.sh"),
new ScriptTask()
.description("Run Sonarqube scan.")
.location(Location.FILE)
.fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh")
.argument(SERVICE_NAME))
.dockerConfiguration(
new DockerConfiguration()
.image(MAVEN_URI)
.volume("/var/run/docker.sock",
"/var/run/docker.sock"))))
.linkedRepositories(REPOSITORY_KEY + " / " + SERVICE_NAME)
.triggers(
new BitbucketServerTrigger())
.planBranchManagement(
new PlanBranchManagement()
.createForVcsBranch()
.delete(
new BranchCleanup()
.whenInactiveInRepositoryAfterDays(
14))
.notificationForCommitters());
}
}

View File

@ -1,8 +0,0 @@
#!/bin/bash
set -e
mkdir -p ~/.ssh
echo "${bamboo_agent_ssh}" | base64 -d >> ~/.ssh/id_rsa
echo "host vector.iqser.com" > ~/.ssh/config
echo " user bamboo-agent" >> ~/.ssh/config
chmod 600 ~/.ssh/config ~/.ssh/id_rsa

View File

@ -1,19 +0,0 @@
#!/bin/bash
set -e
if [[ \"${bamboo_version_tag}\" != \"dev\" ]]
then
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-f ${bamboo_build_working_directory}/pom.xml \
versions:set \
-DnewVersion=${bamboo_version_tag}
${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \
-f ${bamboo_build_working_directory}/pom.xml \
-B clean deploy \
-e -DdeployAtEnd=true \
-Dmaven.wagon.http.ssl.insecure=true \
-Dmaven.wagon.http.ssl.allowall=true \
-Dmaven.wagon.http.ssl.ignore.validity.dates=true \
-DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases
fi

View File

@ -1,84 +0,0 @@
#!/bin/bash
set -e
python3 -m venv build_venv
source build_venv/bin/activate
python3 -m pip install --upgrade pip
pip install poetry
echo "bamboo plan repo branch name: $bamboo_planRepository_branchName"
echo "bamboo version tag: $bamboo_version_tag"
echo "bamboo plan repo 1 branch: $bamboo_planRepository_1_branch"
echo "bamboo build number: $bamboo_buildNumber"
gitVersion=$(git rev-list --tags --max-count=1 | git describe --tags --abbrev=0)
echo "latest version tag in git: $gitVersion"
# update version in poetry to latest version in git if it is lower
# semver regex pattern: (\d+)\.(\d+)\.(\d+)([a-zA-Z\d]*)?-?(dev\d*|post\d*)?
check_poetry_version () {
projectVersion=$(poetry version -s)
tagCount=$(git rev-list --tags --max-count=1 | wc -l)
if [[ $tagCount -gt 0 ]]
then
echo "current version in project: $projectVersion"
if [[ $projectVersion < $gitVersion ]]
then
echo "project version is behind"
echo "setting latest git tag as current version"
poetry version "$gitVersion"
updateVersion=0 # 0 means all is good means true, yes it's weird
elif [[ $projectVersion == $gitVersion ]]
then
echo "project version matches"
echo "keeping the project version"
updateVersion=0
else
echo "project version is higher (aka. has been manually set in pyproject.toml)"
echo "keeping the project version"
updateVersion=1
fi
else
updateVersion=1
fi
return $updateVersion
}
if [[ $bamboo_planRepository_branchName == "master" ]]
then
if check_poetry_version
then
echo "updating version number by one patch increment"
poetry version patch
fi
elif [[ $bamboo_planRepository_branchName =~ ^(release/) ]]
then
if check_poetry_version
then
echo "updating version number by one patch increment"
poetry version patch
fi
else
if check_poetry_version
then
echo "updating version number by one prerelease increment"
# poetry version $(poetry version -s)-dev
fi
fi
if [[ $bamboo_planRepository_branchName =~ ^(master|release/|hotfix/|bugfix/|feature/) ]]
then
newVersion=$(poetry version -s)
else
newVersion="${bamboo_planRepository_1_branch}_${bamboo_buildNumber}"
fi
echo "NEW BUILD on $bamboo_planRepository_branchName with version: $newVersion"
echo "gitTag=$newVersion" > git.tag

View File

@ -1,58 +0,0 @@
#!/bin/bash
set -e
export JAVA_HOME=/usr/bin/sonar-scanner/jre
python3 -m pip install virtualenv
python3 -m venv build_venv
source build_venv/bin/activate
python3 -m pip install --upgrade pip
python3 -m pip install dependency-check
# python3 -m pip install docker-compose
python3 -m pip install coverage
# echo "coverage calculation"
# coverage run -m pytest
# echo "coverage report generation"
# coverage report -m
# coverage xml
SERVICE_NAME=$1
project_name="RED"
pkg_src="pyinfra"
echo "dependency-check:aggregate"
mkdir -p reports
dependency-check --enableExperimental -f JSON -f HTML -f XML \
--disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \
--exclude "build_venv/**" --exclude "**/__pycache__/**" --exclude "bamboo-specs/**"
if [[ -z "${bamboo_repository_pr_key}" ]]
then
echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}"
/usr/bin/sonar-scanner/bin/sonar-scanner -X\
-Dsonar.projectKey=${project_name}_${SERVICE_NAME} \
-Dsonar.sources=${pkg_src} \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.branch.name=${bamboo_planRepository_1_branch} \
-Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \
-Dsonar.python.coverage.reportPaths=reports/coverage.xml
else
echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}"
/usr/bin/sonar-scanner/bin/sonar-scanner \
-Dsonar.projectKey=${project_name}_${SERVICE_NAME} \
-Dsonar.sources=${pkg_src} \
-Dsonar.host.url=https://sonarqube.iqser.com \
-Dsonar.login=${bamboo_sonarqube_api_token_secret} \
-Dsonar.pullrequest.key=${bamboo_repository_pr_key} \
-Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \
-Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \
-Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \
-Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \
-Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \
-Dsonar.python.coverage.reportPaths=reports/coverage.xml
fi

View File

@ -1,16 +0,0 @@
package buildjob;
import com.atlassian.bamboo.specs.api.builders.plan.Plan;
import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException;
import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders;
import org.junit.Test;
public class PlanSpecTest {
@Test
public void checkYourPlanOffline() throws PropertiesValidationException {
Plan plan = new PlanSpec().createBuildPlan();
EntityPropertiesBuilders.build(plan);
}
}

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "pyinfra"
version = "1.5.5"
version = "1.5.6"
description = ""
authors = ["Francisco Schulz <francisco.schulz@iqser.com>"]
license = "All rights reseverd"

View File

@ -1,44 +0,0 @@
attrs==22.2.0 ; python_version >= "3.8" and python_version < "3.9"
azure-core==1.22.1 ; python_version >= "3.8" and python_version < "3.9"
azure-storage-blob==12.9.0 ; python_version >= "3.8" and python_version < "3.9"
bcrypt==4.0.1 ; python_version >= "3.8" and python_version < "3.9"
certifi==2022.12.7 ; python_version >= "3.8" and python_version < "3.9"
cffi==1.15.1 ; python_version >= "3.8" and python_version < "3.9"
charset-normalizer==3.0.1 ; python_version >= "3.8" and python_version < "3.9"
colorama==0.4.6 ; python_version >= "3.8" and python_version < "3.9" and sys_platform == "win32"
cryptography==39.0.1 ; python_version >= "3.8" and python_version < "3.9"
decorator==5.1.1 ; python_version >= "3.8" and python_version < "3.9"
deprecation==2.1.0 ; python_version >= "3.8" and python_version < "3.9"
distro==1.8.0 ; python_version >= "3.8" and python_version < "3.9"
docker-compose==1.29.2 ; python_version >= "3.8" and python_version < "3.9"
docker==6.0.1 ; python_version >= "3.8" and python_version < "3.9"
docker[ssh]==6.0.1 ; python_version >= "3.8" and python_version < "3.9"
dockerpty==0.4.1 ; python_version >= "3.8" and python_version < "3.9"
docopt==0.6.2 ; python_version >= "3.8" and python_version < "3.9"
funcy==1.17 ; python_version >= "3.8" and python_version < "3.9"
idna==3.4 ; python_version >= "3.8" and python_version < "3.9"
isodate==0.6.1 ; python_version >= "3.8" and python_version < "3.9"
jsonschema==3.2.0 ; python_version >= "3.8" and python_version < "3.9"
minio==7.1.3 ; python_version >= "3.8" and python_version < "3.9"
msrest==0.6.21 ; python_version >= "3.8" and python_version < "3.9"
oauthlib==3.2.2 ; python_version >= "3.8" and python_version < "3.9"
packaging==23.0 ; python_version >= "3.8" and python_version < "3.9"
paramiko==3.0.0 ; python_version >= "3.8" and python_version < "3.9"
pika==1.2.0 ; python_version >= "3.8" and python_version < "3.9"
py==1.11.0 ; python_version >= "3.8" and python_version < "3.9"
pycparser==2.21 ; python_version >= "3.8" and python_version < "3.9"
pynacl==1.5.0 ; python_version >= "3.8" and python_version < "3.9"
pyrsistent==0.19.3 ; python_version >= "3.8" and python_version < "3.9"
python-dotenv==0.21.1 ; python_version >= "3.8" and python_version < "3.9"
pywin32==305 ; python_version >= "3.8" and python_version < "3.9" and sys_platform == "win32"
pyyaml==5.4.1 ; python_version >= "3.8" and python_version < "3.9"
requests-oauthlib==1.3.1 ; python_version >= "3.8" and python_version < "3.9"
requests==2.28.2 ; python_version >= "3.8" and python_version < "3.9"
retry==0.9.2 ; python_version >= "3.8" and python_version < "3.9"
setuptools==67.3.1 ; python_version >= "3.8" and python_version < "3.9"
six==1.16.0 ; python_version >= "3.8" and python_version < "3.9"
testcontainers==3.4.2 ; python_version >= "3.8" and python_version < "3.9"
texttable==1.6.7 ; python_version >= "3.8" and python_version < "3.9"
urllib3==1.26.14 ; python_version >= "3.8" and python_version < "3.9"
websocket-client==0.59.0 ; python_version >= "3.8" and python_version < "3.9"
wrapt==1.14.1 ; python_version >= "3.8" and python_version < "3.9"

View File

@ -1,4 +0,0 @@
sonar.exclusions=bamboo-specs/**
sonar.c.file.suffixes=-
sonar.cpp.file.suffixes=-
sonar.objc.file.suffixes=-