Merge pull request #116 from Azure/feature/sdkv2

sdkv2 example tested and working

classical/python-sdk-v2/mlops/azureml/train/components/evaluate.yml

@@ -0,0 +1,24 @@
+# <component>
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+name: evaluate_model
+display_name: evaluate-model
+type: command
+inputs:
+    model_name:
+        type: string
+    model_input:
+        type: uri_folder
+    test_data: 
+        type: uri_folder
+outputs:
+    evaluation_output: 
+        type: uri_folder
+code: ../../../../data-science/src
+environment: azureml:taxi-train-env@latest
+command: >-
+    python evaluate.py 
+    --model_name ${{inputs.model_name}} 
+    --model_input ${{inputs.model_input}} 
+    --test_data ${{inputs.test_data}} 
+    --evaluation_output ${{outputs.evaluation_output}}
+# </component>

classical/python-sdk-v2/mlops/azureml/train/components/prep.yml

@@ -0,0 +1,30 @@
+# <component>
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+name: prep_data
+display_name: prep-data
+type: command
+inputs:
+    raw_data: 
+        type: uri_file
+    enable_monitoring:
+        type: string
+    table_name:
+        type: string
+outputs:
+    train_data: 
+        type: uri_folder
+    val_data: 
+        type: uri_folder
+    test_data:
+        type: uri_folder 
+code: ../../../../data-science/src
+environment: azureml:taxi-train-env@latest
+command: >-
+    python prep.py 
+    --raw_data ${{inputs.raw_data}} 
+    --train_data ${{outputs.train_data}}
+    --val_data ${{outputs.val_data}}
+    --test_data ${{outputs.test_data}}
+    --enable_monitoring ${{inputs.enable_monitoring}}
+    --table_name ${{inputs.table_name}}
+# </component>

classical/python-sdk-v2/mlops/azureml/train/components/register.yml

@@ -0,0 +1,24 @@
+# <component>
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+name: register_model
+display_name: register-model
+type: command
+inputs:
+    model_name: 
+        type: string
+    model_path: 
+        type: uri_folder
+    evaluation_output: 
+        type: uri_folder
+outputs:
+    model_info_output_path: 
+        type: uri_folder
+code: ../../../../data-science/src
+environment: azureml:taxi-train-env@latest
+command: >-
+    python register.py 
+    --model_name ${{inputs.model_name}} 
+    --model_path ${{inputs.model_path}} 
+    --evaluation_output ${{inputs.evaluation_output}}
+    --model_info_output_path ${{outputs.model_info_output_path}}
+# </component>

classical/python-sdk-v2/mlops/azureml/train/components/train.yml

@@ -0,0 +1,18 @@
+# <component>
+$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
+name: train_model
+display_name: train-model
+type: command
+inputs:
+    train_data: 
+        type: uri_folder
+outputs:
+    model_output: 
+        type: uri_folder
+code: ../../../../data-science/src
+environment: azureml:taxi-train-env@latest
+command: >-
+    python train.py 
+    --train_data ${{inputs.train_data}} 
+    --model_output ${{outputs.model_output}}
+# </component>

classical/python-sdk-v2/mlops/azureml/train/evaluate.yml

@@ -1,26 +0,0 @@
-$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
-type: command
-
-name: evaluate_model
-display_name: Evaluate Model
-description: Evaluate Model Component
-version: 5
-inputs:
-  model_name: 
-    type: string
-    default: "taxi-model"
-  model_input: 
-    type: uri_folder    
-  test_data: 
-    type: uri_folder
-outputs:
-  evaluation_output:
-    type: uri_folder
-code: ../../../data-science/src
-environment: azureml:taxi-train-env@latest
-command: >-
-  python evaluate.py 
-  --model_name ${{inputs.model_name}} 
-  --model_input ${{inputs.model_input}} 
-  --test_data ${{inputs.test_data}} 
-  --evaluation_output ${{outputs.evaluation_output}}

classical/python-sdk-v2/mlops/azureml/train/prep.yml

@@ -1,33 +0,0 @@
-$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
-type: command
-
-name: prep_data
-display_name: Prep Data
-description: Data prep component
-version: 1
-inputs:
-  raw_data: 
-    type: uri_folder
-  enable_monitoring:
-    type: string
-    default: false
-  table_name: 
-    type: string
-    default: 'taximonitoring'
-outputs:
-  train_data:
-    type: uri_folder
-  val_data:
-    type: uri_folder
-  test_data:
-    type: uri_folder
-code: ../../../data-science/src
-environment: azureml:taxi-train-env@latest
-command: >-
-  python prep.py 
-  --raw_data ${{inputs.raw_data}} 
-  --train_data ${{outputs.train_data}}
-  --val_data ${{outputs.val_data}}
-  --test_data ${{outputs.test_data}}
-  --enable_monitoring ${{inputs.enable_monitoring}}
-  --table_name ${{inputs.table_name}}

classical/python-sdk-v2/mlops/azureml/train/register.yml

@@ -1,27 +0,0 @@
-$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
-type: command
-
-name: register_model
-display_name: Register Model
-description: Register Model Component
-version: 1
-inputs:
-  model_name: 
-    type: string
-    default: "taxi-model"
-  model_path: 
-    type: uri_folder    
-  evaluation_output: 
-    type: uri_folder
-outputs:
-  model_info_output_path:
-    type: uri_folder
-code: ../../../data-science/src
-environment: azureml:taxi-train-env@latest
-command: >-
-  python register.py 
-  --model_name ${{inputs.model_name}} 
-  --model_path ${{inputs.model_path}} 
-  --evaluation_output ${{inputs.evaluation_output}}
-  --model_info_output_path ${{outputs.model_info_output_path}}
-

classical/python-sdk-v2/mlops/azureml/train/run_pipeline.py

@@ -56,7 +56,7 @@ def main():
     print('current', os.listdir())
 
     # Create pipeline job
-    parent_dir = "mlops/azureml/train"
+    parent_dir = "mlops/azureml/train/components"
 
 
 

classical/python-sdk-v2/mlops/azureml/train/train-env.yml

@@ -1,5 +0,0 @@
-$schema: https://azuremlschemas.azureedge.net/latest/environment.schema.json
-name: taxi-train-env
-image: mcr.microsoft.com/azureml/openmpi3.1.2-ubuntu18.04
-conda_file: ../../../data-science/environment/train-conda.yml
-description: Environment created from a Docker image plus Conda environment to train taxi model.

classical/python-sdk-v2/mlops/azureml/train/train.yml

@@ -1,19 +0,0 @@
-$schema: https://azuremlschemas.azureedge.net/latest/commandComponent.schema.json
-type: command
-
-name: train_model
-display_name: Train Model
-description: Train Model component
-version: 2
-inputs:
-  train_data: 
-    type: uri_folder
-outputs:
-  model_output:
-    type: uri_folder
-code: ../../../data-science/src
-environment: azureml:taxi-train-env@latest
-command: >-
-  python train.py 
-  --train_data ${{inputs.train_data}} 
-  --model_output ${{outputs.model_output}}

classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml

@@ -8,7 +8,8 @@ variables:
 - ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:  
     # 'develop' or feature branches: DEV environment
     - template: ../../config-infra-dev.yml
-
+- name: version
+  value: python-sdk-v2
 
 trigger:
 - none
@@ -39,10 +40,6 @@ stages:
       - template: templates/aml-cli-v2/install-az-cli.yml@mlops-templates
       - template: templates/aml-cli-v2/install-aml-cli.yml@mlops-templates
       - template: templates/aml-cli-v2/connect-to-workspace.yml@mlops-templates
-      - template: templates/aml-cli-v2/register-environment.yml@mlops-templates
-        parameters:
-          environment_name: taxi-train-env
-          environment_file: mlops/azureml/train/train-env.yml
       - template: templates/aml-cli-v2/create-compute.yml@mlops-templates
         parameters:
           cluster_name: cpu-cluster
@@ -50,16 +47,20 @@ stages:
           min_instances: 0
           max_instances: 4
           cluster_tier: LowPriority
-      - template: templates/python-sdk-v2/register-data-asset.yml@mlops-templates
+      - template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
+        parameters:
+          environment_name: taxi-train-env
+          environment_description: "Training Environment for Taxi Pipeline"
+          environment_file: data-science/environment/train-conda.yml
+      - template: templates/${{ variables.version }}/register-data-asset.yml@mlops-templates
         parameters:
           data_name: taxi-data
           data_description: taxi-training-dataset
-          data_path: data/
-          data_type: uri_folder
-      - template: templates/python-sdk-v2/run-pipeline.yml@mlops-templates
+          data_path: data/taxi-data.csv
+          data_type: uri_file
+      - template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
         parameters: 
           pipeline_path: mlops/azureml/train/run_pipeline.py
           data_name: taxi-data
           cluster_name: cpu-cluster
-          enable_monitoring: $(enable_monitoring)
-
+          enable_monitoring: $(enable_monitoring)

classical/python-sdk-v2/mlops/devops-pipelines/deploy-online-endpoint-pipeline.yml

@@ -0,0 +1,60 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+variables:
+- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
+    # 'main' branch: PRD environment
+    - template: ../../config-infra-prod.yml
+- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:  
+    # 'develop' or feature branches: DEV environment
+    - template: ../../config-infra-dev.yml
+- name: version
+  value: python-sdk-v2
+- name: endpoint_name
+  value: taxi-online-$(namespace)$(postfix)$(environment)
+- name: endpoint_type
+  value: online
+
+trigger:
+- none
+
+pool:
+  vmImage: ubuntu-20.04
+
+
+resources:
+  repositories:
+    - repository: mlops-templates  # Template Repo
+      name: Azure/mlops-templates # need to change org name from "Azure" to your own org
+      endpoint: github-connection # need to set up and hardcode
+      type: github
+      ref: main-dec31
+
+stages:
+- stage: CreateOnlineEndpoint
+  displayName: Create/Update Online Endpoint 
+  jobs:
+    - job: DeployOnlineEndpoint
+      steps:
+      - checkout: self
+        path: s/
+      - checkout: mlops-templates
+        path: s/templates/
+      - template: templates/aml-cli-v2/install-az-cli.yml@mlops-templates
+      - template: templates/aml-cli-v2/install-aml-cli.yml@mlops-templates
+      - template: templates/aml-cli-v2/connect-to-workspace.yml@mlops-templates
+      - template: templates/${{ variables.version }}/create-online-endpoint.yml@mlops-templates
+        parameters:
+          endpoint_name: "${{ variables.endpoint_name }}"
+          endpoint_description: "Taxi Online Endpoint"
+          endpoint_auth: "aml_token"
+      - template: templates/${{ variables.version }}/create-online-deployment.yml@mlops-templates
+        parameters:
+          deployment_name: taxi-online-dp
+          endpoint_name: "${{ variables.endpoint_name }}"
+          model_name: "taxi-model@latest"
+          traffic: 100
+      - template: templates/${{ variables.version }}/test-online-endpoint.yml@mlops-templates
+        parameters:
+          endpoint_name: "${{ variables.endpoint_name }}"
+          sample_request: data/taxi-request.json

config-infra-prod.yml

@@ -7,9 +7,9 @@ variables:
   # Global
   ap_vm_image: ubuntu-20.04
 
-  namespace: mlopssdkv2 #Note: A namespace with many characters will cause storage account creation to fail due to storage account names having a limit of 24 characters.
-  postfix: 0971
-  location: westeurope
+  namespace: mlopsv2 #Note: A namespace with many characters will cause storage account creation to fail due to storage account names having a limit of 24 characters.
+  postfix: 001
+  location: uksouth
   environment: prod
   enable_aml_computecluster: true
   enable_monitoring: false