Use PAT in env var/path to authenticate to Azure Pipelines (#82)

2020-08-13 15:16:33 -07:00 · 2020-08-13 15:16:33 -07:00 · dd0b82f5d3
--- a/code/azdocallback/azdocallback.py
+++ b/code/azdocallback/azdocallback.py
@ -1,9 +1,11 @@
+import base64
 import json
 import requests
 import time
 import argparse
 import kfp
 import adal
+import os


 def info(msg, char="#", width=75):
@ -13,18 +15,18 @@ def info(msg, char="#", width=75):
    print(char * width)


-def send_complete_event(callbackinfo, status):  # noqa: E501
+def send_complete_event(callbackinfo, pat, status):  # noqa: E501
    callback_vars = json.loads(callbackinfo.replace("'", '"'))  # noqa: E501
    url = r"{planUri}/{projectId}/_apis/distributedtask/hubs/{hubName}/plans/{planId}/events?api-version=2.0-preview.1".format(                           # noqa: E501
        planUri=callback_vars["PlanUri"], projectId=callback_vars["ProjectId"], hubName=callback_vars["HubName"], planId=callback_vars["PlanId"])          # noqa: E501
    data = {'name': 'TaskCompleted',
            'taskId': callback_vars["TaskInstanceId"], 'jobId': callback_vars["JobId"], 'result': status}   # noqa: E501
-    header = {'Authorization': 'Bearer ' + callback_vars["AuthToken"]}
+    header = {'Authorization': 'Basic ' + pat}
    response = requests.post(url, json=data, headers=header)
    print(response)


-def get_compoenet_status(kfp_host_url, kfp_run_id, token=None):
+def get_component_status(kfp_host_url, kfp_run_id, token=None):
    status = "Suceeded"
    client = kfp.Client(host=kfp_host_url,
                        existing_token=token)
@ -52,6 +54,21 @@ def get_access_token(tenant, clientId, client_secret):
    return token['accessToken']


+def get_pat(pat_env, pat_path_env):
+    if pat_env:
+        # Read PAT from env var
+        pat = os.environ[pat_env]
+    elif pat_path_env:
+        # Read PAT from file
+        with open(os.environ[pat_path_env], 'r') as f:
+            pat = f.readline()
+        f.close
+    else:
+        raise Exception('Please provide a PAT via pat_env or pat_path_env')
+    pat = ":" + pat
+    return str(base64.b64encode(pat.encode("utf-8")), "utf-8")
+
+
 if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Azure DevOps Callback')
    parser.add_argument('-hst', '--kfp_host_url',
@ -65,8 +82,14 @@ if __name__ == "__main__":
                        help='service_principal_id')
    parser.add_argument('-p', '--service_principal_password',
                        help='service_principal_password')
+    parser.add_argument('-ppe', '--pat_path_env',
+                        help='Name of environment variable containing the path to the Azure DevOps PAT')  # noqa: E501
+    parser.add_argument('-pe', '--pat_env',
+                        help='Name of environment variable containing the Azure DevOps PAT')  # noqa: E501
    args = parser.parse_args()

-    status = get_compoenet_status(args.kfp_host_url, args.run_id, get_access_token(    # noqa: E501
+    pat = get_pat(args.pat_env, args.pat_path_env)
+
+    status = get_component_status(args.kfp_host_url, args.run_id, get_access_token(    # noqa: E501
        args.tenant_id, args.service_principal_id, args.service_principal_password))  # noqa: E501
-    send_complete_event(args.azdocallback, status)
+    send_complete_event(args.azdocallback, pat, status)
--- a/code/azdocallback/component.yaml
+++ b/code/azdocallback/component.yaml
@ -6,6 +6,8 @@ inputs:
 - {name: tenant_id, type: Integer}
 - {name: service_principal_id, type: String}
 - {name: service_principal_password, type: String}
+- {name: pat_env, type: String, default: '', description: 'Name of environment variable containing Azure DevOps PAT'}
+- {name: pat_path_env, type: String, default: '', description: 'Name of environment variable containing path to Azure DevOps PAT'}

 implementation:
  container:
@ -17,5 +19,7 @@ implementation:
      '--run_id', {inputValue: run_id},
      '--tenant_id', {inputValue: tenant_id},
      '--service_principal_id', {inputValue: service_principal_id},
-      '--service_principal_password', {inputValue: service_principal_password}
+      '--service_principal_password', {inputValue: service_principal_password},
+      '--pat_env', {inputValue: pat_env},
+      '--pat_path_env', {inputValue: pat_path_env}
    ]
--- a/code/azure/devops/queue_pipeline/component.yaml
+++ b/code/azure/devops/queue_pipeline/component.yaml
@ -6,6 +6,8 @@ inputs:
    - {name: organization, type: String, description: 'Azure DevOps organization'}
    - {name: project, type: String, description: 'Azure DevOps project'}
    - {name: id, type: Integer, description: 'Azure Pipeline definition id'}
+    - {name: pat_env, type: String, default: '', description: 'Name of environment variable containing Azure DevOps PAT'}
+    - {name: pat_path_env, type: String, default: '', description: 'Name of environment variable containing path to Azure DevOps PAT'}
    - {name: source_branch, type: String, default: '', description: 'Source branch for the pipeline'}
    - {name: source_version, type: String, default: '', description: 'Source version for the pipeline'}
    - {name: parameters, type: String, default: '', description: 'Parameters for the pipeline'}
@ -19,6 +21,8 @@ implementation:
            --organization, {inputValue: organization},
            --project, {inputValue: project},
            --id, {inputValue: id},
+            --pat_env, {inputValue: pat_env},
+            --pat_path_env, {inputValue: pat_path_env},
            --source_branch, {inputValue: source_branch},
            --source_version, {inputValue: source_version},
            --parameters, {inputValue: parameters}
--- a/code/azure/devops/queue_pipeline/readme.md
+++ b/code/azure/devops/queue_pipeline/readme.md
@ -9,6 +9,8 @@ This task enables you to queue an [Azure Pipelines](https://docs.microsoft.com/e
 |organization|string|Y|The Azure DevOps organization that contains the pipeline to be queued. https[]()://dev.azure.com/`organization`/project/_build?definitionId=id|
 |project|string|Y|The Azure DevOps project that contains the pipeline to be queued. https[]()://dev.azure.com/organization/`project`/_build?definitionId=id|
 |id|string|Y|The id of the pipeline definition to queue. Shown in the url as *pipelineId* or *definitionId*. https[]()://dev.azure.com/organization/project/_build?definitionId=`id`|
+|pat_env|string|one of pat_env or pat_path_env|The name of the environment variable containing the PAT for Azure Pipelines authentication|
+|pat_path_env|string|one of pat_env or pat_path_env|The name of the environment variable containing a path to the PAT for Azure Pipelines authentication|
 |sourch_branch|string||The branch of the source code for queuing the pipeline.|
 |source_version|string||The version (e.g. commit id) of the source code for queuing the pipeline.|
 |parameters|string||Json serialized string of key-values pairs e.g. `{ 'x': '1', 'y': '2' }`. These values can be accessed as `$(x)` and `$(y)` in the Azure Pipelines pipeline.|
@ -25,6 +27,7 @@ import kfp
 from kfp import components
 from kfp.dsl.extensions.kubernetes import use_secret
 import kfp.compiler as compiler
+from kubernetes import client as k8s_client

 component_root = os.path.join(os.path.dirname(
    os.path.abspath(__file__)), ".")
@ -34,14 +37,24 @@ queue_pipeline_op = components.load_component_from_file(os.path.join(component_r
 def queue_az_pipeline():
    secret_name = "azdopat"
    secret_path = "/app/secrets"
+    pat_path_env = "PAT_PATH"
+    secret_file_path_in_volume = "azdopat"
    organization = # organization
    project = # project
    pipeline_id = # id
    queue_task = queue_pipeline_op(
        organization=organization,
        project=project,
-        id=pipeline_id
+        id=pipeline_id,
+        pat_path_env=pat_path_env
    ).apply(use_secret(secret_name=secret_name, secret_volume_mount_path=secret_path))

+    # path separator workaround when compiling on windows
+    queue_task.container.add_env_variable(
+        k8s_client.V1EnvVar(
+            name=pat_path_env,
+            value=secret_path + "/" + secret_file_path_in_volume
+        ))
+
 kfp.Client().create_run_from_pipeline_func(queue_az_pipeline, arguments={})
 ```
--- a/code/azure/devops/queue_pipeline/src/queue_pipeline.py
+++ b/code/azure/devops/queue_pipeline/src/queue_pipeline.py
@ -26,6 +26,7 @@ POSSIBILITY OF SUCH DAMAGE.
 """

 import argparse
+import os
 from azure.devops.connection import Connection
 from msrest.authentication import BasicAuthentication

@ -79,6 +80,10 @@ def main():
    parser.add_argument('-i', '--id',
                        required=True,
                        help='Id of the pipeline definition')
+    parser.add_argument('-ppe', '--pat_path_env',
+                        help='Name of environment variable containing the path to the Azure DevOps PAT')  # noqa: E501
+    parser.add_argument('-pe', '--pat_env',
+                        help='Name of environment variable containing the Azure DevOps PAT')  # noqa: E501
    parser.add_argument(
        '--source_branch', help='Source branch for the pipeline')
    parser.add_argument(
@ -86,13 +91,19 @@ def main():
    parser.add_argument(
        '--parameters', help='Parameters for the pipeline')

-    # Read PAT from file
-    with open("/app/secrets/azdopat", 'r') as f:
-        pat = f.readline()
-    f.close()
-
    args = parser.parse_args()

+    if args.pat_env:
+        # Read PAT from env var
+        pat = os.environ[args.pat_env]
+    elif args.pat_path_env:
+        # Read PAT from file
+        with open(os.environ[args.pat_path_env], 'r') as f:
+            pat = f.readline()
+        f.close
+    else:
+        raise Exception('Please provide a PAT via pat_env or pat_path_env')
+
    client = get_client(args.organization, pat)
    build = define_build(args.id,
                         args.source_branch,
--- a/code/pipelineazdo.py
+++ b/code/pipelineazdo.py
@ -7,7 +7,7 @@ import kfp.compiler as compiler
 import kfp.components as components
 from kfp.azure import use_azure_secret
 from kubernetes.client.models import V1EnvVar
-from utils.kfp_helper import use_databricks_secret, use_image, use_kfp_host_secret
+from utils.kfp_helper import use_databricks_secret, use_image, use_kfp_host_secret, use_secret_var


 persistent_volume_path = '/mnt/azure'
@ -63,7 +63,12 @@ def tacosandburritos_train(
                              run_id=dsl.RUN_ID_PLACEHOLDER,
                              tenant_id="$(AZ_TENANT_ID)",
                              service_principal_id="$(AZ_CLIENT_ID)",
-                              service_principal_password="$(AZ_CLIENT_SECRET)").apply(use_azure_secret()).apply(use_kfp_host_secret()).apply(use_image(exit_image_name))  # noqa: E501
+                              service_principal_password="$(AZ_CLIENT_SECRET)",
+                              pat_env="PAT_ENV"
+                              ).apply(use_azure_secret()
+                              ).apply(use_kfp_host_secret()
+                              ).apply(use_image(exit_image_name)
+                              ).apply(use_secret_var("azdopat", "PAT_ENV", "azdopat"))

    with dsl.ExitHandler(exit_op=exit_handler_op):

--- a/code/utils/kfp_helper.py
+++ b/code/utils/kfp_helper.py
@ -62,6 +62,27 @@ def use_kfp_host_secret(secret_name='kfp-host-secret'):
    return _use_kfp_host_secret


+def use_secret_var(secret_name, env_var_name, secret_key):
+    def _use_secret_var(task):
+        from kubernetes import client as k8s_client
+        (
+            task.container
+                .add_env_variable(
+                    k8s_client.V1EnvVar(
+                        name=env_var_name,
+                        value_from=k8s_client.V1EnvVarSource(
+                            secret_key_ref=k8s_client.V1SecretKeySelector(
+                                name=secret_name,
+                                key=secret_key
+                            )
+                        )
+                    )
+                )
+        )
+        return task
+    return _use_secret_var
+
+
 def use_image(image_name):
    def _use_image(task):
        task.image = image_name