From 2f20ec4ecdf349e9cc74ee644f588529bbc62785 Mon Sep 17 00:00:00 2001
From: Francisco Schulz <francisco.schulz@knecon.com>
Date: Mon, 6 May 2024 17:10:56 +0200
Subject: [PATCH] feat: update CI and add DVC job

---
 .gitlab-ci.yml | 61 +++++++++++++++++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f14ce1a..19e715b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,32 +1,51 @@
 include:
   - project: "Gitlab/gitlab"
-    ref: 0.3.0
-    file: "/ci-templates/research/dvc-versioning-build-release.gitlab-ci.yml"
+    file: "/ci-templates/research/versioning-build-test-release.gitlab-ci.yml"
     # file: "/ci-templates/research/versioning-build+azure_model-test-release.gitlab-ci.yml"
 
 variables:
   NEXUS_PROJECT_DIR: red
   IMAGENAME: "${CI_PROJECT_NAME}"
 
-#################################
-# temp. disable integration tests, b/c they don't cover the CV analysis case yet
-trigger integration tests:
+stages:
+  - data
+  - setup
+  - unit-tests
+  - versioning
+  - build
+  - integration-tests
+  - release
+
+###########
+# PULL DATA
+dvc-pull:
+  image: python:3.10
+  stage: data
+  script:
+  - pip install dvc
+  - pip install dvc-azure
+  - dvc remote add -d -f ${DVC_REMOTE_NAME} azure://${DVC_AZURE_BLOB_STORE_NAME}/
+  - dvc remote modify ${DVC_REMOTE_NAME} connection_string "${DVC_AZURE_BLOB_STORE_CONNECTION_STRING}"
+  - dvc pull
+  rules: !reference [.default-rules-config, rules]
+  artifacts:
+    name: "${CI_PROJECT_DIR}-dvc-files"
+    paths:
+    - data/*
+    - models/*
+    expire_in: 1 day
+
+###################
+# INTEGRATION TESTS
+trigger-integration-tests:
   rules:
-    - when: never
+    - when: never # temp. disable integration tests
 
-release build:
-  stage: release
+#########
+# RELEASE
+release:
+  extends: .release
   needs:
-  - job: set custom version
-    artifacts: true
-    optional: true
-  - job: calculate patch version
-    artifacts: true
-    optional: true
-  - job: calculate minor version
-    artifacts: true
-    optional: true
-  - job: build docker nexus
-    artifacts: true
-#################################
-
+    - !reference [.needs-versioning, needs] # leave this line as is
+    - job: dvc-pull
+      artifacts: true