Use PAT in env var/path to authenticate to Azure Pipelines (#82)

code/azdocallback/azdocallback.py

@@ -1,9 +1,11 @@
+import base64
 import json
 import requests
 import time
 import argparse
 import kfp
 import adal
+import os
 
 
 def info(msg, char="#", width=75):
@@ -13,18 +15,18 @@ def info(msg, char="#", width=75):
     print(char * width)
 
 
-def send_complete_event(callbackinfo, status):  # noqa: E501
+def send_complete_event(callbackinfo, pat, status):  # noqa: E501
     callback_vars = json.loads(callbackinfo.replace("'", '"'))  # noqa: E501
     url = r"{planUri}/{projectId}/_apis/distributedtask/hubs/{hubName}/plans/{planId}/events?api-version=2.0-preview.1".format(                           # noqa: E501
         planUri=callback_vars["PlanUri"], projectId=callback_vars["ProjectId"], hubName=callback_vars["HubName"], planId=callback_vars["PlanId"])          # noqa: E501
     data = {'name': 'TaskCompleted',
             'taskId': callback_vars["TaskInstanceId"], 'jobId': callback_vars["JobId"], 'result': status}   # noqa: E501
-    header = {'Authorization': 'Bearer ' + callback_vars["AuthToken"]}
+    header = {'Authorization': 'Basic ' + pat}
     response = requests.post(url, json=data, headers=header)
     print(response)
 
 
-def get_compoenet_status(kfp_host_url, kfp_run_id, token=None):
+def get_component_status(kfp_host_url, kfp_run_id, token=None):
     status = "Suceeded"
     client = kfp.Client(host=kfp_host_url,
                         existing_token=token)
@@ -52,6 +54,21 @@ def get_access_token(tenant, clientId, client_secret):
     return token['accessToken']
 
 
+def get_pat(pat_env, pat_path_env):
+    if pat_env:
+        # Read PAT from env var
+        pat = os.environ[pat_env]
+    elif pat_path_env:
+        # Read PAT from file
+        with open(os.environ[pat_path_env], 'r') as f:
+            pat = f.readline()
+        f.close
+    else:
+        raise Exception('Please provide a PAT via pat_env or pat_path_env')
+    pat = ":" + pat
+    return str(base64.b64encode(pat.encode("utf-8")), "utf-8")
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Azure DevOps Callback')
     parser.add_argument('-hst', '--kfp_host_url',
@@ -65,8 +82,14 @@ if __name__ == "__main__":
                         help='service_principal_id')
     parser.add_argument('-p', '--service_principal_password',
                         help='service_principal_password')
+    parser.add_argument('-ppe', '--pat_path_env',
+                        help='Name of environment variable containing the path to the Azure DevOps PAT')  # noqa: E501
+    parser.add_argument('-pe', '--pat_env',
+                        help='Name of environment variable containing the Azure DevOps PAT')  # noqa: E501
     args = parser.parse_args()
 
-    status = get_compoenet_status(args.kfp_host_url, args.run_id, get_access_token(    # noqa: E501
+    pat = get_pat(args.pat_env, args.pat_path_env)
+
+    status = get_component_status(args.kfp_host_url, args.run_id, get_access_token(    # noqa: E501
         args.tenant_id, args.service_principal_id, args.service_principal_password))  # noqa: E501
-    send_complete_event(args.azdocallback, status)
+    send_complete_event(args.azdocallback, pat, status)

code/azdocallback/component.yaml

@@ -6,6 +6,8 @@ inputs:
 - {name: tenant_id, type: Integer}
 - {name: service_principal_id, type: String}
 - {name: service_principal_password, type: String}
+- {name: pat_env, type: String, default: '', description: 'Name of environment variable containing Azure DevOps PAT'}
+- {name: pat_path_env, type: String, default: '', description: 'Name of environment variable containing path to Azure DevOps PAT'}
 
 implementation:
   container:
@@ -17,5 +19,7 @@ implementation:
       '--run_id', {inputValue: run_id},
       '--tenant_id', {inputValue: tenant_id},
       '--service_principal_id', {inputValue: service_principal_id},
-      '--service_principal_password', {inputValue: service_principal_password}
+      '--service_principal_password', {inputValue: service_principal_password},
+      '--pat_env', {inputValue: pat_env},
+      '--pat_path_env', {inputValue: pat_path_env}
     ]

code/azure/devops/queue_pipeline/component.yaml

@@ -6,6 +6,8 @@ inputs:
     - {name: organization, type: String, description: 'Azure DevOps organization'}
     - {name: project, type: String, description: 'Azure DevOps project'}
     - {name: id, type: Integer, description: 'Azure Pipeline definition id'}
+    - {name: pat_env, type: String, default: '', description: 'Name of environment variable containing Azure DevOps PAT'}
+    - {name: pat_path_env, type: String, default: '', description: 'Name of environment variable containing path to Azure DevOps PAT'}
     - {name: source_branch, type: String, default: '', description: 'Source branch for the pipeline'}
     - {name: source_version, type: String, default: '', description: 'Source version for the pipeline'}
     - {name: parameters, type: String, default: '', description: 'Parameters for the pipeline'}
@@ -19,6 +21,8 @@ implementation:
             --organization, {inputValue: organization},
             --project, {inputValue: project},
             --id, {inputValue: id},
+            --pat_env, {inputValue: pat_env},
+            --pat_path_env, {inputValue: pat_path_env},
             --source_branch, {inputValue: source_branch},
             --source_version, {inputValue: source_version},
             --parameters, {inputValue: parameters}

code/azure/devops/queue_pipeline/readme.md

@@ -9,6 +9,8 @@ This task enables you to queue an [Azure Pipelines](https://docs.microsoft.com/e
 |organization|string|Y|The Azure DevOps organization that contains the pipeline to be queued. https[]()://dev.azure.com/`organization`/project/_build?definitionId=id|
 |project|string|Y|The Azure DevOps project that contains the pipeline to be queued. https[]()://dev.azure.com/organization/`project`/_build?definitionId=id|
 |id|string|Y|The id of the pipeline definition to queue. Shown in the url as *pipelineId* or *definitionId*. https[]()://dev.azure.com/organization/project/_build?definitionId=`id`|
+|pat_env|string|one of pat_env or pat_path_env|The name of the environment variable containing the PAT for Azure Pipelines authentication|
+|pat_path_env|string|one of pat_env or pat_path_env|The name of the environment variable containing a path to the PAT for Azure Pipelines authentication|
 |sourch_branch|string||The branch of the source code for queuing the pipeline.|
 |source_version|string||The version (e.g. commit id) of the source code for queuing the pipeline.|
 |parameters|string||Json serialized string of key-values pairs e.g. `{ 'x': '1', 'y': '2' }`. These values can be accessed as `$(x)` and `$(y)` in the Azure Pipelines pipeline.|
@@ -25,6 +27,7 @@ import kfp
 from kfp import components
 from kfp.dsl.extensions.kubernetes import use_secret
 import kfp.compiler as compiler
+from kubernetes import client as k8s_client
 
 component_root = os.path.join(os.path.dirname(
     os.path.abspath(__file__)), ".")
@@ -34,14 +37,24 @@ queue_pipeline_op = components.load_component_from_file(os.path.join(component_r
 def queue_az_pipeline():
     secret_name = "azdopat"
     secret_path = "/app/secrets"
+    pat_path_env = "PAT_PATH"
+    secret_file_path_in_volume = "azdopat"
     organization = # organization
     project = # project
     pipeline_id = # id
     queue_task = queue_pipeline_op(
         organization=organization,
         project=project,
-        id=pipeline_id
+        id=pipeline_id,
+        pat_path_env=pat_path_env
     ).apply(use_secret(secret_name=secret_name, secret_volume_mount_path=secret_path))
 
+    # path separator workaround when compiling on windows
+    queue_task.container.add_env_variable(
+        k8s_client.V1EnvVar(
+            name=pat_path_env,
+            value=secret_path + "/" + secret_file_path_in_volume
+        ))
+
 kfp.Client().create_run_from_pipeline_func(queue_az_pipeline, arguments={})
 ```

code/azure/devops/queue_pipeline/src/queue_pipeline.py

@@ -26,6 +26,7 @@ POSSIBILITY OF SUCH DAMAGE.
 """
 
 import argparse
+import os
 from azure.devops.connection import Connection
 from msrest.authentication import BasicAuthentication
 
@@ -79,6 +80,10 @@ def main():
     parser.add_argument('-i', '--id',
                         required=True,
                         help='Id of the pipeline definition')
+    parser.add_argument('-ppe', '--pat_path_env',
+                        help='Name of environment variable containing the path to the Azure DevOps PAT')  # noqa: E501
+    parser.add_argument('-pe', '--pat_env',
+                        help='Name of environment variable containing the Azure DevOps PAT')  # noqa: E501
     parser.add_argument(
         '--source_branch', help='Source branch for the pipeline')
     parser.add_argument(
@@ -86,13 +91,19 @@ def main():
     parser.add_argument(
         '--parameters', help='Parameters for the pipeline')
 
-    # Read PAT from file
-    with open("/app/secrets/azdopat", 'r') as f:
-        pat = f.readline()
-    f.close()
-
     args = parser.parse_args()
 
+    if args.pat_env:
+        # Read PAT from env var
+        pat = os.environ[args.pat_env]
+    elif args.pat_path_env:
+        # Read PAT from file
+        with open(os.environ[args.pat_path_env], 'r') as f:
+            pat = f.readline()
+        f.close
+    else:
+        raise Exception('Please provide a PAT via pat_env or pat_path_env')
+
     client = get_client(args.organization, pat)
     build = define_build(args.id,
                          args.source_branch,

code/pipelineazdo.py

@@ -7,7 +7,7 @@ import kfp.compiler as compiler
 import kfp.components as components
 from kfp.azure import use_azure_secret
 from kubernetes.client.models import V1EnvVar
-from utils.kfp_helper import use_databricks_secret, use_image, use_kfp_host_secret
+from utils.kfp_helper import use_databricks_secret, use_image, use_kfp_host_secret, use_secret_var
 
 
 persistent_volume_path = '/mnt/azure'
@@ -63,7 +63,12 @@ def tacosandburritos_train(
                               run_id=dsl.RUN_ID_PLACEHOLDER,
                               tenant_id="$(AZ_TENANT_ID)",
                               service_principal_id="$(AZ_CLIENT_ID)",
-                              service_principal_password="$(AZ_CLIENT_SECRET)").apply(use_azure_secret()).apply(use_kfp_host_secret()).apply(use_image(exit_image_name))  # noqa: E501
+                              service_principal_password="$(AZ_CLIENT_SECRET)",
+                              pat_env="PAT_ENV"
+                              ).apply(use_azure_secret()
+                              ).apply(use_kfp_host_secret()
+                              ).apply(use_image(exit_image_name)
+                              ).apply(use_secret_var("azdopat", "PAT_ENV", "azdopat"))
 
     with dsl.ExitHandler(exit_op=exit_handler_op):
 

code/utils/kfp_helper.py

@@ -62,6 +62,27 @@ def use_kfp_host_secret(secret_name='kfp-host-secret'):
     return _use_kfp_host_secret
 
 
+def use_secret_var(secret_name, env_var_name, secret_key):
+    def _use_secret_var(task):
+        from kubernetes import client as k8s_client
+        (
+            task.container
+                .add_env_variable(
+                    k8s_client.V1EnvVar(
+                        name=env_var_name,
+                        value_from=k8s_client.V1EnvVarSource(
+                            secret_key_ref=k8s_client.V1SecretKeySelector(
+                                name=secret_name,
+                                key=secret_key
+                            )
+                        )
+                    )
+                )
+        )
+        return task
+    return _use_secret_var
+
+
 def use_image(image_name):
     def _use_image(task):
         task.image = image_name