Use correct tags in CI pipeline (#71)

2020-07-20 11:53:42 -07:00 · 2020-07-20 11:53:42 -07:00 · 0884feaaf9
--- a/.azdo/build-container-template.yml
+++ b/.azdo/build-container-template.yml
@ -9,6 +9,7 @@ parameters:
    type: string
  - name: containerReferences # space (' ') delimited set of files or folders in sourceLocation to check for changes
    type: string
+    default: '.'
  - name: acr # name of Azure Container Registry
    type: string
  - name: repository # name of container repository
@ -53,4 +54,8 @@ steps:
        fi
        set +x
        echo "setting IMAGE_TAG output for task ${{ parameters.taskname }}"
-        echo "##vso[task.setvariable variable=IMAGE_TAG;isOutput=true]$imageTag"
+        echo "##vso[task.setvariable variable=IMAGE_TAG;isOutput=true]$imageTag"
+
+        # write a file containing the image tag
+        mkdir -p $(Pipeline.Workspace)/image_tags
+        echo "$imageTag" > $(Pipeline.Workspace)/image_tags/${{ parameters.taskname }}
--- a/.azdo/ci.yaml
+++ b/.azdo/ci.yaml
@ -35,82 +35,66 @@ stages:
      BUILD_ALL_IMAGES: true
    jobs:
      - job:
-        #condition: and(succeeded(), eq(variables['BUILD_ALL_IMAGES '], 'true'))
        steps:
-        - task: Docker@2
-          displayName: Build and Push MLFlow Project
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/mlflow-project/'
+            dockerFile: 'code/mlflow-project/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/mlflowproject'
-            command: 'buildAndPush'
-            Dockerfile: 'code/mlflow-project/Dockerfile'
-            buildContext: 'code/mlflow-project/'
-            tags: 'latest'
-        
-        # Disabling building all other images, uncomment to include specific component from code\pipelineazdo.py
-        - task: Docker@2
-          displayName: Build and Push AzdoCallback Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'mlflowproject_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/azdocallback/'
+            dockerFile: 'code/azdocallback/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/azdocallback'
-            command: 'buildAndPush'
-            Dockerfile: 'code/azdocallback/Dockerfile'
-            buildContext: 'code/azdocallback/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Preprocess Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'azdocallback_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/preprocess/'
+            dockerFile: 'code/preprocess/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/preprocess'
-            command: 'buildAndPush'
-            Dockerfile: 'code/preprocess/Dockerfile'
-            buildContext: 'code/preprocess/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Register Artifacts Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'preprocess_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/register-artifacts/'
+            dockerFile: 'code/register-artifacts/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/registerartifacts'
-            command: 'buildAndPush'
-            Dockerfile: 'code/register-artifacts/Dockerfile'
-            buildContext: 'code/register-artifacts/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Register MLFlow Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'registerartifacts_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/register-mlflow/'
+            dockerFile: 'code/register-mlflow/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/register-mlflow'
-            command: 'buildAndPush'
-            Dockerfile: 'code/register-mlflow/Dockerfile'
-            buildContext: 'code/register-mlflow/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Register Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'registermlflow_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/register/'
+            dockerFile: 'code/register/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/register'
-            command: 'buildAndPush'
-            Dockerfile: 'code/register/Dockerfile'
-            buildContext: 'code/register/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Training Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'register_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/training/'
+            dockerFile: 'code/training/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/training'
-            command: 'buildAndPush'
-            Dockerfile: 'code/training/Dockerfile'
-            buildContext: 'code/training/'
-            tags: 'latest'
-        - task: Docker@2
-          displayName: Build and Push Databricks Image
-          inputs:
-            containerRegistry: 'kubeflow-azdo-sample-acr-sc'
+            taskname: 'training_container'
+        - template: build-container-template.yml
+          parameters:
+            sourceLocation: 'code/databricks/'
+            dockerFile: 'code/databricks/Dockerfile'
+            acr: 'kubeflowyoacr'
            repository: 'mexicanfood/databricks-notebook'
-            command: 'buildAndPush'
-            Dockerfile: 'code/databricks/Dockerfile'
-            buildContext: 'code/databricks/'
-            tags: 'latest'
+            taskname: 'databricks_container'
+        - publish: $(Pipeline.Workspace)/image_tags
+          artifact: image_tags
+          displayName: "Publish image_tags"

  - stage: "Build_Upload_Run_Kubeflow_Pipeline"
    displayName: 'Build, Upload, and Run Kubeflow Pipeline'
@ -119,6 +103,9 @@ stages:
    jobs:
      - job: "Upload_Pipeline"
        steps:
+        - download: current
+          artifact: image_tags
+          displayName: "Download image_tags"
        - task: UsePythonVersion@0
          inputs:
            versionSpec: '3.7'
@ -133,8 +120,19 @@ stages:
          displayName: "Compile Pipeline"
          inputs:
            targetType: 'inline'
-            script: 'python code/pipelineazdo.py'
-       
+            script: |
+              if [ -d "$(Pipeline.Workspace)/image_tags" ]; then
+                export MLFLOWPROJECT_TAG=`cat $(Pipeline.Workspace)/image_tags/mlflowproject_container`
+                export AZDOCALLBACK_TAG=`cat $(Pipeline.Workspace)/image_tags/azdocallback_container`
+                export PREPROCESS_TAG=`cat $(Pipeline.Workspace)/image_tags/preprocess_container`
+                export REGISTERARTIFACTS_TAG=`cat $(Pipeline.Workspace)/image_tags/registerartifacts_container`
+                export REGISTERMLFLOW_TAG=`cat $(Pipeline.Workspace)/image_tags/registermlflow_container`
+                export REGISTER_TAG=`cat $(Pipeline.Workspace)/image_tags/register_container`
+                export TRAINING_TAG=`cat $(Pipeline.Workspace)/image_tags/training_container`
+                export DATABRICKS_TAG=`cat $(Pipeline.Workspace)/image_tags/databricks_container`
+              fi
+
+              python code/pipelineazdo.py
        - task: KubeflowUploadPipeline@0
          displayName: "Upload Pipeline to KubeFlow" 
          inputs:
--- a/code/pipelineazdo.py
+++ b/code/pipelineazdo.py
@ -1,4 +1,5 @@
 "KubeFlow Pipeline with AzureDevops Callback"
+import os
 from kubernetes import client as k8s_client
 import kfp.dsl as dsl
 import kfp.compiler as compiler
@ -78,7 +79,7 @@ def tacosandburritos_train(

    exit_op = dsl.ContainerOp(
        name='Exit Handler',
-        image=image_repo_name + '/azdocallback:latest',
+        image=image_repo_name + '/azdocallback:%s' % (os.getenv('AZDOCALLBACK_TAG') or 'latest'),  # noqa: E501
        command=['python'],
        arguments=[
            '/scripts/azdocallback.py',
@ -95,7 +96,7 @@ def tacosandburritos_train(

        operations['mlflowproject'] = dsl.ContainerOp(
            name='Run MLflow Project on Azure Databricks',
-            image=image_repo_name + '/mlflowproject:latest',
+            image=image_repo_name + '/mlflowproject:%s' % (os.getenv('MLFLOWPROJECT_TAG') or 'latest'),  # noqa: E501
            command=['python'],
            arguments=[
                '/scripts/run.py',
@ -106,7 +107,7 @@ def tacosandburritos_train(

        # operations['preprocess'] = dsl.ContainerOp(
        #     name='preprocess',
-        #     image=image_repo_name + '/preprocess:latest',
+        #     image=image_repo_name + '/preprocess:%s' % (os.getenv('PREPROCESS_TAG') or 'latest'),  # noqa: E501
        #     command=['python'],
        #     arguments=[
        #         '/scripts/data.py',
@ -125,7 +126,7 @@ def tacosandburritos_train(
        # with dsl.ParallelFor([{'epochs': 1, 'lr': 0.0001}, {'epochs': 1, 'lr': 0.0002}]) as item:  # noqa: E501
        operations['training'] = dsl.ContainerOp(
            name="training",
-            image=image_repo_name + '/training:latest',
+            image=image_repo_name + '/training:%s' % (os.getenv('TRAINING_TAG') or 'latest'),  # noqa: E501
            command=['python'],
            arguments=[
                '/scripts/train.py',
@ -161,7 +162,7 @@ def tacosandburritos_train(
        # register kubeflow artifcats model
        operations['register to kubeflow'] = dsl.ContainerOp(
            name='register to kubeflow',
-            image=image_repo_name + '/registerartifacts:latest',
+            image=image_repo_name + '/registerartifacts:%s' % (os.getenv('REGISTERARTIFACTS_TAG') or 'latest'),  # noqa: E501
            command=['python'],
            arguments=[
                '/scripts/registerartifacts.py',
@ -178,7 +179,7 @@ def tacosandburritos_train(
        # register model
        operations['register to AML'] = dsl.ContainerOp(
            name='register to AML',
-            image=image_repo_name + '/register:latest',
+            image=image_repo_name + '/register:%s' % (os.getenv('REGISTER_TAG') or 'latest'),  # noqa: E501
            command=['python'],
            arguments=[
                '/scripts/register.py',
@ -199,7 +200,7 @@ def tacosandburritos_train(
        # register model to mlflow
        operations['register to mlflow'] = dsl.ContainerOp(
            name='register to mlflow',
-            image=image_repo_name + '/register-mlflow:latest',
+            image=image_repo_name + '/register-mlflow:%s' % (os.getenv('REGISTERMLFLOW_TAG') or 'latest'),  # noqa: E501
            command=['python'],
            arguments=[
                '/scripts/register.py',