From 66616c64c54e2f7526dc8c212dcede4de5e91a8e Mon Sep 17 00:00:00 2001 From: Isaac Riley Date: Thu, 3 Mar 2022 15:02:56 +0100 Subject: [PATCH] fully containerize app; first full attempt and still needs testing --- bamboo-specs/pom.xml | 40 ++++ .../src/main/java/buildjob/PlanSpec.java | 175 ++++++++++++++++++ .../main/resources/scripts/create-licence.sh | 19 ++ .../main/resources/scripts/docker-build.sh | 19 ++ .../src/main/resources/scripts/git-tag.sh | 9 + .../src/main/resources/scripts/sonar-scan.sh | 61 ++++++ .../src/test/java/buildjob/PlanSpecTest.java | 16 ++ .../target/classes/buildjob/PlanSpec.class | Bin 0 -> 11147 bytes .../target/classes/scripts/create-licence.sh | 19 ++ .../target/classes/scripts/docker-build.sh | 19 ++ .../target/classes/scripts/git-tag.sh | 9 + .../target/classes/scripts/sonar-scan.sh | 61 ++++++ .../test-classes/buildjob/PlanSpecTest.class | Bin 0 -> 940 bytes requirements.txt | 9 +- src/run_service.py | 81 ++++++++ vidocp/config.py | 36 ++++ vidocp/locations.py | 14 ++ vidocp/table_parsig.py | 1 - vidocp/utils/logging.py | 22 +++ 19 files changed, 607 insertions(+), 3 deletions(-) create mode 100644 bamboo-specs/pom.xml create mode 100644 bamboo-specs/src/main/java/buildjob/PlanSpec.java create mode 100755 bamboo-specs/src/main/resources/scripts/create-licence.sh create mode 100755 bamboo-specs/src/main/resources/scripts/docker-build.sh create mode 100755 bamboo-specs/src/main/resources/scripts/git-tag.sh create mode 100755 bamboo-specs/src/main/resources/scripts/sonar-scan.sh create mode 100644 bamboo-specs/src/test/java/buildjob/PlanSpecTest.java create mode 100644 bamboo-specs/target/classes/buildjob/PlanSpec.class create mode 100644 bamboo-specs/target/classes/scripts/create-licence.sh create mode 100644 bamboo-specs/target/classes/scripts/docker-build.sh create mode 100644 bamboo-specs/target/classes/scripts/git-tag.sh create mode 100644 bamboo-specs/target/classes/scripts/sonar-scan.sh create mode 100644 bamboo-specs/target/test-classes/buildjob/PlanSpecTest.class create mode 100644 vidocp/config.py create mode 100644 vidocp/locations.py create mode 100644 vidocp/utils/logging.py diff --git a/bamboo-specs/pom.xml b/bamboo-specs/pom.xml new file mode 100644 index 0000000..40bc09e --- /dev/null +++ b/bamboo-specs/pom.xml @@ -0,0 +1,40 @@ + + 4.0.0 + + + com.atlassian.bamboo + bamboo-specs-parent + 7.1.2 + + + + bamboo-specs + 1.0.0-SNAPSHOT + jar + + + true + + + + + com.atlassian.bamboo + bamboo-specs-api + + + com.atlassian.bamboo + bamboo-specs + + + + + junit + junit + test + + + + + + \ No newline at end of file diff --git a/bamboo-specs/src/main/java/buildjob/PlanSpec.java b/bamboo-specs/src/main/java/buildjob/PlanSpec.java new file mode 100644 index 0000000..564578c --- /dev/null +++ b/bamboo-specs/src/main/java/buildjob/PlanSpec.java @@ -0,0 +1,175 @@ +package buildjob; + +import com.atlassian.bamboo.specs.api.BambooSpec; +import com.atlassian.bamboo.specs.api.builders.BambooKey; +import com.atlassian.bamboo.specs.api.builders.docker.DockerConfiguration; +import com.atlassian.bamboo.specs.api.builders.permission.PermissionType; +import com.atlassian.bamboo.specs.api.builders.permission.Permissions; +import com.atlassian.bamboo.specs.api.builders.permission.PlanPermissions; +import com.atlassian.bamboo.specs.api.builders.plan.Job; +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.builders.plan.PlanIdentifier; +import com.atlassian.bamboo.specs.api.builders.plan.Stage; +import com.atlassian.bamboo.specs.api.builders.plan.branches.BranchCleanup; +import com.atlassian.bamboo.specs.api.builders.plan.branches.PlanBranchManagement; +import com.atlassian.bamboo.specs.api.builders.project.Project; +import com.atlassian.bamboo.specs.builders.task.CheckoutItem; +import com.atlassian.bamboo.specs.builders.task.InjectVariablesTask; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.builders.task.VcsCheckoutTask; +import com.atlassian.bamboo.specs.builders.task.CleanWorkingDirectoryTask; +import com.atlassian.bamboo.specs.builders.task.VcsTagTask; +import com.atlassian.bamboo.specs.builders.trigger.BitbucketServerTrigger; +import com.atlassian.bamboo.specs.model.task.InjectVariablesScope; +import com.atlassian.bamboo.specs.api.builders.Variable; +import com.atlassian.bamboo.specs.util.BambooServer; +import com.atlassian.bamboo.specs.builders.task.ScriptTask; +import com.atlassian.bamboo.specs.model.task.ScriptTaskProperties.Location; + +/** + * Plan configuration for Bamboo. + * Learn more on: https://confluence.atlassian.com/display/BAMBOO/Bamboo+Specs + */ +@BambooSpec +public class PlanSpec { + + private static final String SERVICE_NAME = "vidocp-base"; + private static final String SERVICE_NAME_BASE = "vidocp-base"; + + private static final String SERVICE_KEY = SERVICE_NAME.toUpperCase().replaceAll("-","").replaceAll("_",""); + + /** + * Run main to publish plan on Bamboo + */ + public static void main(final String[] args) throws Exception { + //By default credentials are read from the '.credentials' file. + BambooServer bambooServer = new BambooServer("http://localhost:8085"); + + Plan plan = new PlanSpec().createDockerBuildPlan(); + bambooServer.publish(plan); + PlanPermissions planPermission = new PlanSpec().createPlanPermission(plan.getIdentifier()); + bambooServer.publish(planPermission); + } + + private PlanPermissions createPlanPermission(PlanIdentifier planIdentifier) { + Permissions permission = new Permissions() + .userPermissions("atlbamboo", PermissionType.EDIT, PermissionType.VIEW, PermissionType.ADMIN, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("research", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("Development", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .groupPermissions("QA", PermissionType.EDIT, PermissionType.VIEW, PermissionType.CLONE, PermissionType.BUILD) + .loggedInUserPermissions(PermissionType.VIEW) + .anonymousUserPermissionView(); + return new PlanPermissions(planIdentifier.getProjectKey(), planIdentifier.getPlanKey()).permissions(permission); + } + + private Project project() { + return new Project() + .name("RED") + .key(new BambooKey("RED")); + } + + public Plan createDockerBuildPlan() { + return new Plan( + project(), + SERVICE_NAME, new BambooKey(SERVICE_KEY)) + .description("Docker build for vidocp.") + // .variables() + .stages(new Stage("Build Stage") + .jobs( + new Job("Build Job", new BambooKey("BUILD")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Build Docker container.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/docker-build.sh") + .argument(SERVICE_NAME + " " + SERVICE_NAME_BASE)) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock")), + new Job("Sonar Job", new BambooKey("SONAR")) + .tasks( + new CleanWorkingDirectoryTask() + .description("Clean working directory.") + .enabled(true), + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Set config and keys.") + .inlineBody("mkdir -p ~/.ssh\n" + + "echo \"${bamboo.bamboo_agent_ssh}\" | base64 -d >> ~/.ssh/id_rsa\n" + + "echo \"host vector.iqser.com\" > ~/.ssh/config\n" + + "echo \" user bamboo-agent\" >> ~/.ssh/config\n" + + "chmod 600 ~/.ssh/config ~/.ssh/id_rsa"), + new ScriptTask() + .description("Run Sonarqube scan.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/sonar-scan.sh") + .argument(SERVICE_NAME)) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:4.2.0") + .volume("/var/run/docker.sock", "/var/run/docker.sock"))), + new Stage("Licence Stage") + .jobs( + new Job("Git Tag Job", new BambooKey("GITTAG")) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build git tag.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/git-tag.sh"), + new InjectVariablesTask() + .description("Inject git tag.") + .path("git.tag") + .namespace("g") + .scope(InjectVariablesScope.LOCAL), + new VcsTagTask() + .description("${bamboo.g.gitTag}") + .tagName("${bamboo.g.gitTag}") + .defaultRepository()) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/release_build:2.7.0")), + new Job("Licence Job", new BambooKey("LICENCE")) + .enabled(false) + .tasks( + new VcsCheckoutTask() + .description("Checkout default repository.") + .checkoutItems(new CheckoutItem().defaultRepository()), + new ScriptTask() + .description("Build licence.") + .location(Location.FILE) + .fileFromPath("bamboo-specs/src/main/resources/scripts/create-licence.sh")) + .dockerConfiguration( + new DockerConfiguration() + .image("nexus.iqser.com:5001/infra/maven:3.6.2-jdk-13-3.0.0") + .volume("/etc/maven/settings.xml", "/usr/share/maven/ref/settings.xml") + .volume("/var/run/docker.sock", "/var/run/docker.sock")))) + .linkedRepositories("RR / " + SERVICE_NAME) + .triggers(new BitbucketServerTrigger()) + .planBranchManagement(new PlanBranchManagement() + .createForVcsBranch() + .delete(new BranchCleanup() + .whenInactiveInRepositoryAfterDays(14)) + .notificationForCommitters()); + } + + +} diff --git a/bamboo-specs/src/main/resources/scripts/create-licence.sh b/bamboo-specs/src/main/resources/scripts/create-licence.sh new file mode 100755 index 0000000..a9054cd --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/create-licence.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +if [[ \"${bamboo_version_tag}\" != \"dev\" ]] +then + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + versions:set \ + -DnewVersion=${bamboo_version_tag} + + ${bamboo_capability_system_builder_mvn3_Maven_3}/bin/mvn \ + -f ${bamboo_build_working_directory}/pom.xml \ + -B clean deploy \ + -e -DdeployAtEnd=true \ + -Dmaven.wagon.http.ssl.insecure=true \ + -Dmaven.wagon.http.ssl.allowall=true \ + -Dmaven.wagon.http.ssl.ignore.validity.dates=true \ + -DaltDeploymentRepository=iqser_release::default::https://nexus.iqser.com/repository/gin4-platform-releases +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/docker-build.sh b/bamboo-specs/src/main/resources/scripts/docker-build.sh new file mode 100755 index 0000000..f17638f --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/docker-build.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -e + +SERVICE_NAME=$1 +SERVICE_NAME_BASE=$2 + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +pip install dvc +pip install 'dvc[ssh]' +dvc pull + +echo "index-url = https://${bamboo_nexus_user}:${bamboo_nexus_password}@nexus.iqser.com/repository/python-combind/simple" >> pip.conf +docker build -f Dockerfile_base -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} . +docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} . +echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 +docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} diff --git a/bamboo-specs/src/main/resources/scripts/git-tag.sh b/bamboo-specs/src/main/resources/scripts/git-tag.sh new file mode 100755 index 0000000..2005666 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/git-tag.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +if [[ "${bamboo_version_tag}" = "dev" ]] +then + echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag +else + echo "gitTag=${bamboo_version_tag}" > git.tag +fi \ No newline at end of file diff --git a/bamboo-specs/src/main/resources/scripts/sonar-scan.sh b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh new file mode 100755 index 0000000..76f6830 --- /dev/null +++ b/bamboo-specs/src/main/resources/scripts/sonar-scan.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +export JAVA_HOME=/usr/bin/sonar-scanner/jre + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +echo "dev setup for unit test and coverage" + +pip install -e . +pip install -r requirements.txt + + +echo "DVC pull step" +dvc pull + +echo "coverage calculation" +coverage run -m pytest +echo "coverage report generation" +coverage report -m +coverage xml + +SERVICE_NAME=$1 + +echo "dependency-check:aggregate" +mkdir -p reports +dependency-check --enableExperimental -f JSON -f HTML -f XML \ + --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ + --exclude "build_venv/**" --exclude "**/__pycache__/**" + +if [[ -z "${bamboo_repository_pr_key}" ]] +then + echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" + /usr/bin/sonar-scanner/bin/sonar-scanner -X\ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=src,ner \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml + +else + echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" + /usr/bin/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=src,ner \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ + -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ + -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml +fi diff --git a/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java new file mode 100644 index 0000000..fada379 --- /dev/null +++ b/bamboo-specs/src/test/java/buildjob/PlanSpecTest.java @@ -0,0 +1,16 @@ +package buildjob; + + +import com.atlassian.bamboo.specs.api.builders.plan.Plan; +import com.atlassian.bamboo.specs.api.exceptions.PropertiesValidationException; +import com.atlassian.bamboo.specs.api.util.EntityPropertiesBuilders; +import org.junit.Test; + +public class PlanSpecTest { + @Test + public void checkYourPlanOffline() throws PropertiesValidationException { + Plan plan = new PlanSpec().createDockerBuildPlan(); + + EntityPropertiesBuilders.build(plan); + } +} \ No newline at end of file diff --git a/bamboo-specs/target/classes/buildjob/PlanSpec.class b/bamboo-specs/target/classes/buildjob/PlanSpec.class new file mode 100644 index 0000000000000000000000000000000000000000..a4a7656025829f1541688cc264145a1645ad2efb GIT binary patch literal 11147 zcmcgy349#YdH;Va*}GZ~U{>ZZN3%GP*ck1~x3NG$mu-P88?7uONHC+-NE&H(X4#pQ zZ453q;Xcfj1W168i-ZQdS+x-+ZrrAILQ9XNO-a%-X`8m`ku*I?(*K)T?P_hY?Ara+ zKY4fO&3oVb?(4m;|NP6Z16VCj2N4jQK9n<4!{hc)d|OJl5*Z^IM7?0)xIU@JIWijG zIy7!1U4fS9>Ko|p=-S!awxvrD=_yS~xLMO0Z4sQ(VOx%?Tke3K${8Uv2o_A5!*(*$ zG^9HQx0?-QXM0(opy4<$hIpbR0=3qL&;UZO3#>ak0nOd ziBo8nr?)K%fejj3aG4;O$ql7UXG}1-v}#a&w(WLVuIcXCmbIxyu4y?r)t*Cn>1BK!^fgWpt0chVG*kgshznL# zTK5h*Bsf(qE+I>PRuAT#DGp+rptWi>nFXl$(+OiME(#$bXrQ_JJPKmFpzWmK_V39U z4d}-X4cCCqx4x^hyI;_8LSz@QRZv?L*y}WW&;vWr-L*pvT(4oL8kpDCxuv^Tjp!PN zypfKct-W1pWLSgYjkIs??&(y7k7^i$NpG5SjBH6f)S;*6@HN-{A0U5XnqFjp(-Oi2 zLs`~v3_Y71Q$(aSSc-^+qggwbnMFki8Ah*8W70_3nKb=D2=oSltJ~BnS3?eyg0oWg z=%_K=ZEZi61$Q$DoU{(m#hnxkhu&u^c5B#!n*`_TmTm1x+c~Ek+X2(qC1|h3&lOo3 zpT)73AZ{j{6T!I5zG+qaaBC31ENH137$JB*t>hOq+=knIf6myUlBKJRJ+%cgKg@C& z`M3{vYPd^53^PI~7tZl8_TnBDU_UG{tM}q1xK+!yv_u8%y|_=q{dj;HnkmYHhIe=M zIF#0v$?(K9u3Ucf_nylx$wF{>)s)T^;>h$8zJe>N2Ug@x8I>v<@pXbaR*T~Mi{b@^ zqQ*ZiSXZ@@mah(DzX~2tF{AG5qDS42k7)QPK1P3U>1igVTjp@1HOQxYH{$wi4QWwP z-@B?(av}~ z3iVe6E2~yOwJzc6qpSg6B}!f<9rXsncm*qi_*Fr3P0+5iheF6R|M6jYk1Puh;@6do zzAV^02c;*(>L6aLL4RG{nTYrEBZ${42PIExs%-I=bfgU3+F@rWSUGo^Sw;;zyGOwZ zEQslxACXU z4Uz@&)V>U#Zsnw{0@RAqC|`FBS82lEVPRA?s}i0Q@#XQBlERMKcuCb};kbDXiTksPz5OD^0y#WR)yH!$f-I`1*e~B_&I*Tf>-%Q8va;i!!PShl-y}WK9sHT za$3g5mY_)GXjHb$XF}Xpzz)y8b6HQ8VYW&y3JF`nG0WJUb4nJtVNG-MinwWwWOcHY z(abS+dR*MFI<_*_%!M*vlc0o1y_u#R3a+WuY^p@EB_!N#QG8O*#9fgF^Dnw%~XcD(#~W&9QFu4pUOa-e!-iz*Tl zw#^$7wxwsIDxDH`$LA%s_O|tfgdOt6zMK{H-rkrSGNMjWw_+hVhtalL({dE(CU2oK ztXJebP0klK$?M1MAsT~OgE17bu8~kL=cIl_8H9Vxq+uluKMxJb5+Y3Y0AQBGkdOWKz*1Fn{Ku4+=NoUx!>EV!(K|MrZgm=|_i;|z5( z2Lz56e`T4n!6tUV{I}5=oWgu_BqZ$12RRhuh8&TVnyix5WOIgoOVBrmwpZlhyCOt> zuBBaho2}R6QnguwDkYr^Pf;4AMJ^8tJMkMTP(3RM97A#<$HIu(+n;DUJ0$;O>AJ!=%EWQ~!sX>17yt!P9>>D3%+fUoBj1{ zyx;#0%8iw?r;>{-ENnO3p&Tqe{$HaW%=wB$LC!Mx@lPK^DiLS4)4Z z)WMP6f-5U{ z^j7u}HSWo@md*M~#ZU!+K1yCsC6@`yK85{0xm}YxY=JcYwJo_xE%|pwhpRN*&=asqE#b} z3I@*#!U?bT+sLYp*`OL!_g!M@as;Vky6c*0W59Gw_P*LIQtMSfl>72Z*@x1&cK@0L zp>Z3n+hXzFp)OSzPEek#IuDwwdZ){nso!w{d((BiyHkgAFc0(j6y#F?AwGwXdFJ&K zzMIc?8cyNNQ~9*cTMe8Q3FUEG_Ab8PM^;RuX)qF>!it`0 z$lw%K=dotX@=Nl#bVp%;zb~IcNAL0`HCWp69FN2pNpHp(Si!42E4j%kKCSl9xY*wS zo3NQ1g^5fzt|S6y;5=M~9uK7;wxE}&1k}9=z9XAt5#cH}=cloi2wffN3mroLVBG-> zBnImb;M&CCyaV`9VsQQe+>jUy9zZgY$4F#6IL9 zFYw?%Njd6MTo4&)$Afr?{CEM2@Gu@B#DF>z@i|0DN|3ha)Uht(53UV4pIr@#r8+rWRVH_oI-&9lIK%>;dxg;;K!?LVV;(1xoC~NYv?tpB_%N0rp(#c618)Z{ox(`dI z_v6aE^p>&zYO;Se*AqU@+kKoVoE_92PfKdT``U86KdA7YE%oa?so&5jLzH@%_!UF* zGE9YAI9S&xV|*B|KO`2lPckGZ%K>LvcJb$)yxcS``}phDg{S7_w!GYxmwWT_K+(YD zcHBTq+)Oj;rnA42_PQBo(Z4OBXInu_?Z(yEhCbf!N$~DXKSr<}E(UnRUD1HhPZ)05QdBbG^QW8O0qOfEI zY+28%ELY%0*~}{|Td_;7!EU)8dt{imLdG#BY2Mp#af{r9eR2o(%01|n2YLPCS==U5 zxIVo)e-KqFo zT_eBG#LIPyIes4btvsj~^ygku4DR376uL$&N@WO> pip.conf +docker build -f Dockerfile_base -t nexus.iqser.com:5001/red/$SERVICE_NAME_BASE:${bamboo_version_tag} . +docker build -f Dockerfile -t nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} --build-arg VERSION_TAG=${bamboo_version_tag} . +echo "${bamboo_nexus_password}" | docker login --username "${bamboo_nexus_user}" --password-stdin nexus.iqser.com:5001 +docker push nexus.iqser.com:5001/red/$SERVICE_NAME:${bamboo_version_tag} diff --git a/bamboo-specs/target/classes/scripts/git-tag.sh b/bamboo-specs/target/classes/scripts/git-tag.sh new file mode 100644 index 0000000..2005666 --- /dev/null +++ b/bamboo-specs/target/classes/scripts/git-tag.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +if [[ "${bamboo_version_tag}" = "dev" ]] +then + echo "gitTag=${bamboo_planRepository_1_branch}_${bamboo_buildNumber}" > git.tag +else + echo "gitTag=${bamboo_version_tag}" > git.tag +fi \ No newline at end of file diff --git a/bamboo-specs/target/classes/scripts/sonar-scan.sh b/bamboo-specs/target/classes/scripts/sonar-scan.sh new file mode 100644 index 0000000..76f6830 --- /dev/null +++ b/bamboo-specs/target/classes/scripts/sonar-scan.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +export JAVA_HOME=/usr/bin/sonar-scanner/jre + +python3 -m venv build_venv +source build_venv/bin/activate +python3 -m pip install --upgrade pip + +echo "dev setup for unit test and coverage" + +pip install -e . +pip install -r requirements.txt + + +echo "DVC pull step" +dvc pull + +echo "coverage calculation" +coverage run -m pytest +echo "coverage report generation" +coverage report -m +coverage xml + +SERVICE_NAME=$1 + +echo "dependency-check:aggregate" +mkdir -p reports +dependency-check --enableExperimental -f JSON -f HTML -f XML \ + --disableAssembly -s . -o reports --project $SERVICE_NAME --exclude ".git/**" --exclude "venv/**" \ + --exclude "build_venv/**" --exclude "**/__pycache__/**" + +if [[ -z "${bamboo_repository_pr_key}" ]] +then + echo "Sonar Scan for branch: ${bamboo_planRepository_1_branch}" + /usr/bin/sonar-scanner/bin/sonar-scanner -X\ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=src,ner \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.branch.name=${bamboo_planRepository_1_branch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml + +else + echo "Sonar Scan for PR with key1: ${bamboo_repository_pr_key}" + /usr/bin/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=RED_$SERVICE_NAME \ + -Dsonar.sources=src,ner \ + -Dsonar.host.url=https://sonarqube.iqser.com \ + -Dsonar.login=${bamboo_sonarqube_api_token_secret} \ + -Dsonar.pullrequest.key=${bamboo_repository_pr_key} \ + -Dsonar.pullrequest.branch=${bamboo_repository_pr_sourceBranch} \ + -Dsonar.pullrequest.base=${bamboo_repository_pr_targetBranch} \ + -Dsonar.dependencyCheck.jsonReportPath=reports/dependency-check-report.json \ + -Dsonar.dependencyCheck.xmlReportPath=reports/dependency-check-report.xml \ + -Dsonar.dependencyCheck.htmlReportPath=reports/dependency-check-report.html \ + -Dsonar.python.coverage.reportPaths=reports/coverage.xml +fi diff --git a/bamboo-specs/target/test-classes/buildjob/PlanSpecTest.class b/bamboo-specs/target/test-classes/buildjob/PlanSpecTest.class new file mode 100644 index 0000000000000000000000000000000000000000..1bc1310875c92a2f1aa7d0cc42ec2920bc1aa74e GIT binary patch literal 940 zcma)4+iuf95IvJPHAzD*B`uU&DHkaPiG2W`QXz3EyaZa&DyWZZXWMRSdo6p7fWP8} zB9Qn1J_<4Gmgb?L^krsecjnBQ+4Jl7_n!bBreUy)EKEWjiv66k%a# zLK%%@rd@n;>DdFF6D|+`*?d6gg$F7;+%-k!SZ(g@MT|>mY%h(8NO8 zNanegi3nsI7$b5XA{R2%LY=0It<0nJ#NX_>=?IU`O<}qX7Q;5nBUG=nlUdho_OE&$I^kj!EK^oGP~j8HldaE@U{CC8 z_m6PlOZnVj>z;u+=Giun6WCwmh|uEKD%yPZIO<@5PcHuwdk6W3TU6M0f1tN}gr%=6 o?FZj*?fn$KRdQ)Ee2W3vWt*krqLViB47rNi3`@AfU=`f`19cevQvd(} literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index 0714f67..bf6d0be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,13 @@ opencv-python~=4.5.5.62 numpy~=1.22.1 pdf2image~=1.16.0 matplotlib~=3.5.1 -imutils==0.5.4 -iteration-utilities==0.11.0 +imutils~=0.5.4 +iteration-utilities~=0.11.0 dvc==2.9.3 dvc[ssh] scipy~=1.8.0 +PyYAML~=6.0 +Flask~=2.0 +waitress~=2.0 +pytest~=6.2 +envyaml~=1.8 \ No newline at end of file diff --git a/src/run_service.py b/src/run_service.py index b5fcf6d..b7e4a16 100644 --- a/src/run_service.py +++ b/src/run_service.py @@ -1,6 +1,87 @@ import argparse import json import logging +from flask import Flask, request, jsonify +from waitress import serve + +from vidocp.utils import preprocess #TODO +from vidocp.table_parsing import parse_table#, detect_tables_in_pdf +from vidocp.redaction_detection import find_redactions#, detect_redactions_in_pdf +from vidocp.layout_parsing import parse_layout#, detect_layout_in_pdf #TODO +from vidocp.figure_detection import detect_figures#, detect_figures_in_pdf #TODO +from vidocp.utils.logging import logger +from vidocp.config import CONFIG + + +def suppress_user_warnings(): + import warnings + warnings.filterwarnings("ignore") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--warnings", action="store_true", default=False) + args = parser.parse_args() + + return args + + +def main(args): + + if not args.warnings: + suppress_user_warnings() + + run_server() + + +def run_server(): + app = Flask(__name__) + + @app.route("/", methods=["POST"]) + def predict_request(): + def inner(): + data = request.data + logger.info(f"<3 Received data.") + #section_text = json.loads(data) + logger.info(f"Processing data. <3") + predictions = make_predictions(data) + return jsonify({"result": predictions}) + try: + return inner() + except Exception as err: + logger.warning("Analysis failed") + logger.exception(err) + resp = jsonify("Analysis failed") + resp.status_code = 500 + return resp + + @app.route("/status", methods=["GET"]) + def status(): + response = "OK" + return jsonify(response) + + #predictor = initialize_predictor() + #logger.info("<3 Predictor ready.") + + mode = CONFIG.webserver.mode + if mode == "development": + app.run(host=CONFIG.webserver.host, port=CONFIG.webserver.port, debug=True) + elif mode == "production": + serve(app, host=CONFIG.webserver.host, port=CONFIG.webserver.port) + + +def make_predictions(pdf_data, page_index): + pdf = preprocess(pdf_data[page_index]) + tables = parse_table(pdf) + redactions = find_redactions(pdf) + layout = parse_layout(pdf) + figure = detect_figures(pdf) + return jsonify({"tables": tables, + "redactions": redactions, + "layout": layout, + "figure": figure}) + + """ from tkinter import W diff --git a/vidocp/config.py b/vidocp/config.py new file mode 100644 index 0000000..eaf36ce --- /dev/null +++ b/vidocp/config.py @@ -0,0 +1,36 @@ +"""Implements a config object with dot-indexing syntax.""" + + +from envyaml import EnvYAML +from vidocp.locations import CONFIG_FILE + + +def _get_item_and_maybe_make_dotindexable(container, item): + ret = container[item] + return DotIndexable(ret) if isinstance(ret, dict) else ret + + +class DotIndexable: + def __init__(self, x): + self.x = x + + def __getattr__(self, item): + return _get_item_and_maybe_make_dotindexable(self.x, item) + + def __setitem__(self, key, value): + self.x[key] = value + + def __repr__(self): + return self.x.__repr__() + + +class Config: + def __init__(self, config_path): + self.__config = EnvYAML(config_path) + + def __getattr__(self, item): + if item in self.__config: + return _get_item_and_maybe_make_dotindexable(self.__config, item) + + def __getitem__(self, item): + return self.__getattr__(item) \ No newline at end of file diff --git a/vidocp/locations.py b/vidocp/locations.py new file mode 100644 index 0000000..20e64a8 --- /dev/null +++ b/vidocp/locations.py @@ -0,0 +1,14 @@ +"""Defines constant paths relative to the module root path.""" + + +from os import path + +MODULE_DIR = path.dirname(path.abspath(__file__)) +PACKAGE_ROOT_DIR = path.dirname(MODULE_DIR) + +CONFIG_FILE = path.join(PACKAGE_ROOT_DIR, "config.yaml") +LOG_FILE = "/tmp/log.log" + +DVC_DATA_DIR = path.join(PACKAGE_ROOT_DIR, "data") + +TEST_DATA_DIR = path.join(PACKAGE_ROOT_DIR, "test", "test_data") \ No newline at end of file diff --git a/vidocp/table_parsig.py b/vidocp/table_parsig.py index 00a3d36..099830e 100644 --- a/vidocp/table_parsig.py +++ b/vidocp/table_parsig.py @@ -75,7 +75,6 @@ def annotate_table(image, parsed_tables): return image - def filter_unconnected_cells(stats): filtered_cells = [] # print(stats) diff --git a/vidocp/utils/logging.py b/vidocp/utils/logging.py new file mode 100644 index 0000000..f4c6d6b --- /dev/null +++ b/vidocp/utils/logging.py @@ -0,0 +1,22 @@ +"""Defines the default logger for the service.""" + + +import sys +import logging + +from vidocp.config import CONFIG + +def get_logger(): + logger = logging.getLogger(__name__) + logger.setLevel(logging.getLevelName(CONFIG.service.logging_level)) + formatter = logging.Formatter(fmt="%(asctime)s %(levelname)s: %(message)s", datefmt="%d.%m.%Y - %H:%M:%S") + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.getLevelName(CONFIG.service.logging_level)) + ch.setFormatter(formatter) + + logger.addHandler(ch) + logger.propagate = False + return logger + +logger = get_logger() \ No newline at end of file