From a8be488f6384c22e98518466fcd7debd779fe5cc Mon Sep 17 00:00:00 2001 From: Maggie Mhanna Date: Thu, 8 Dec 2022 13:26:20 +0000 Subject: [PATCH 1/6] update ado pipelines to match changes in mlops-templates --- .../devops-pipelines/deploy-model-training-pipeline.yml | 2 +- .../devops-pipelines/deploy-online-endpoint-pipeline.yml | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml index 8b1efa9..c8237ce 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml @@ -51,7 +51,7 @@ stages: parameters: environment_name: taxi-train-env environment_description: "Training Environment for Taxi Pipeline" - environment_file: data-science/environment/train-conda.yml + conda_file: data-science/environment/train-conda.yml - template: templates/${{ variables.version }}/register-data-asset.yml@mlops-templates parameters: data_name: taxi-data diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml index 2f9e1f8..b47917a 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml @@ -12,8 +12,6 @@ variables: value: python-sdk-v2 - name: endpoint_name value: taxi-online-$(namespace)$(postfix)$(environment) -- name: endpoint_type - value: online trigger: - none @@ -47,13 +45,13 @@ stages: parameters: endpoint_name: "${{ variables.endpoint_name }}" endpoint_description: "Taxi Online Endpoint" - endpoint_auth: "aml_token" + auth_mode: "aml_token" - template: templates/${{ variables.version }}/create-online-deployment.yml@mlops-templates parameters: deployment_name: taxi-online-dp endpoint_name: "${{ variables.endpoint_name }}" - model_name: "taxi-model@latest" - traffic: 100 + model_path: "taxi-model@latest" + traffic_allocation: 100 - template: templates/${{ variables.version }}/test-online-endpoint.yml@mlops-templates parameters: endpoint_name: "${{ variables.endpoint_name }}" From 8c768f11eba3e2eccf76496339efc69a9800d438 Mon Sep 17 00:00:00 2001 From: Maggie Mhanna Date: Thu, 8 Dec 2022 13:26:50 +0000 Subject: [PATCH 2/6] adding ado batch endpoint deployment pipeline --- .../deploy-batch-endpoint-pipeline.yml | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml new file mode 100644 index 0000000..5a3bf0a --- /dev/null +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +variables: +- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}: + # 'main' branch: PRD environment + - template: ../../config-infra-prod.yml +- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}: + # 'develop' or feature branches: DEV environment + - template: ../../config-infra-dev.yml +- name: version + value: python-sdk-v2 +- name: endpoint_name + value: taxi-batch-$(namespace)$(postfix)$(environment) + +trigger: +- none + +pool: + vmImage: ubuntu-20.04 + + +resources: + repositories: + - repository: mlops-templates # Template Repo + name: Azure/mlops-templates # need to change org name from "Azure" to your own org + endpoint: github-connection # need to set up and hardcode + type: github + ref: feature/sdkv2 + + +stages: +- stage: CreateBatchEndpoint + displayName: Create/Update Batch Endpoint + jobs: + - job: DeployBatchEndpoint + steps: + - checkout: self + path: s/ + - checkout: mlops-templates + path: s/templates/ + - template: templates/aml-cli-v2/install-az-cli.yml@mlops-templates + - template: templates/aml-cli-v2/install-aml-cli.yml@mlops-templates + - template: templates/aml-cli-v2/connect-to-workspace.yml@mlops-templates + - template: templates/aml-cli-v2/create-compute.yml@mlops-templates + parameters: + cluster_name: batch-cluster # name must match cluster name in deployment file below + size: STANDARD_DS3_V2 + min_instances: 0 + max_instances: 5 + cluster_tier: dedicated + - template: templates/${{ variables.version }}/create-batch-endpoint.yml@mlops-templates + parameters: + endpoint_name: "${{ variables.endpoint_name }}" + endpoint_description: "Taxi batch endpoint" + auth_mode: "aad_token" + - template: templates/${{ variables.version }}/create-batch-deployment.yml@mlops-templates + parameters: + deployment_name: taxi-batch-dp + deployment_description: "Taxi batch deployment" + endpoint_name: "${{ variables.endpoint_name }}" + model_path: "taxi-model@latest" + compute: batch-cluster + - template: templates/${{ variables.version }}/test-batch-endpoint.yml@mlops-templates + parameters: + endpoint_name: "${{ variables.endpoint_name }}" + sample_request: data/taxi-batch.csv + request_type: uri_file #either uri_folder or uri_file \ No newline at end of file From 9a7645348e25e99faffa24ea2a08a48e61fee17b Mon Sep 17 00:00:00 2001 From: Cindy Weng <8880364+cindyweng@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:55:33 +0000 Subject: [PATCH 3/6] Update deploy-batch-endpoint-pipeline.yml --- .../mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml index 5a3bf0a..9788999 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-batch-endpoint-pipeline.yml @@ -26,7 +26,7 @@ resources: name: Azure/mlops-templates # need to change org name from "Azure" to your own org endpoint: github-connection # need to set up and hardcode type: github - ref: feature/sdkv2 + ref: main-dec31 stages: @@ -65,4 +65,4 @@ stages: parameters: endpoint_name: "${{ variables.endpoint_name }}" sample_request: data/taxi-batch.csv - request_type: uri_file #either uri_folder or uri_file \ No newline at end of file + request_type: uri_file #either uri_folder or uri_file From 78b3236a88848dad11737b0cd4e00e2fa31c96e0 Mon Sep 17 00:00:00 2001 From: Cindy Weng <8880364+cindyweng@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:56:18 +0000 Subject: [PATCH 4/6] Update deploy-model-training-pipeline.yml --- .../devops-pipelines/deploy-model-training-pipeline.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml index c8237ce..935a1cd 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml @@ -51,7 +51,8 @@ stages: parameters: environment_name: taxi-train-env environment_description: "Training Environment for Taxi Pipeline" - conda_file: data-science/environment/train-conda.yml + environment_path: data-science/environment/train-conda.yml + build_type: conda - template: templates/${{ variables.version }}/register-data-asset.yml@mlops-templates parameters: data_name: taxi-data @@ -63,4 +64,4 @@ stages: pipeline_path: mlops/azureml/train/run_pipeline.py data_name: taxi-data cluster_name: cpu-cluster - enable_monitoring: $(enable_monitoring) \ No newline at end of file + enable_monitoring: $(enable_monitoring) From d7a350730b934763479a57c7d1304c46291b167c Mon Sep 17 00:00:00 2001 From: Cindy Weng <8880364+cindyweng@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:56:40 +0000 Subject: [PATCH 5/6] Update deploy-model-training-pipeline.yml --- .../mlops/devops-pipelines/deploy-model-training-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml index 935a1cd..4b219cb 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml @@ -24,7 +24,7 @@ resources: name: Azure/mlops-templates # need to change org name from "Azure" to your own org endpoint: github-connection # need to set up and hardcode type: github - ref: feature/sdkv2 + ref: main-dec31 stages: - stage: DeployTrainingPipeline From 8a39c333daa791d7797c32456ddcd6179c48b6d5 Mon Sep 17 00:00:00 2001 From: Cindy Weng <8880364+cindyweng@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:57:18 +0000 Subject: [PATCH 6/6] Update deploy-online-endpoint-pipeline.yml --- .../devops-pipelines/deploy-online-endpoint-pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml index b47917a..a1a6cee 100644 --- a/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml +++ b/classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml @@ -26,7 +26,7 @@ resources: name: Azure/mlops-templates # need to change org name from "Azure" to your own org endpoint: github-connection # need to set up and hardcode type: github - ref: feature/sdkv2 + ref: main-dec31 stages: - stage: CreateOnlineEndpoint @@ -55,4 +55,4 @@ stages: - template: templates/${{ variables.version }}/test-online-endpoint.yml@mlops-templates parameters: endpoint_name: "${{ variables.endpoint_name }}" - sample_request: data/taxi-request.json \ No newline at end of file + sample_request: data/taxi-request.json