Adding Build Scripts

2019-08-27 03:41:40 -04:00 · 2019-08-27 03:41:40 -04:00 · 6d3df22838
--- a/.ci/python-keras-scoring.yml
+++ b/.ci/python-keras-scoring.yml
@ -0,0 +1,106 @@
+# BatchScoringDeepLearningModelsWithAML Pipeline 
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+    
+variables:
+- group: AzureKeyVault
+
+jobs:
+- job: BatchScoringDeepLearningModelsWithAMLJob
+  timeoutInMinutes: 300
+  cancelTimeoutInMinutes: 2
+  pool:
+    vmImage: 'Ubuntu-16.04'
+
+  steps:
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      which conda
+      conda env create -f environment.yml
+      conda activate batchscoringdl_aml
+      conda env list  
+      echo Login Azure Account
+      az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret)
+    displayName: 'Initial Step'
+  
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      conda env list  
+      cd notebooks
+      echo Execute 01_local_testing.ipynb
+      papermill 01_local_testing.ipynb 01_local_testing_output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '01_local_testing.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      conda env list  
+      cd notebooks
+      echo Execute 02_setup_aml.ipynb
+      papermill 02_setup_aml.ipynb 02_setup_aml_output.ipynb --log-output --no-progress-bar -k python3 \
+      -p subscription_id $(subscriptionid) \
+      -p resource_group $(azurergname) \
+      -p workspace_name $(workspacename) \
+      -p workspace_region $(azureregion) \
+      -p storage_account_name $(azurestorage)
+    displayName: '02_setup_aml.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      conda env list  
+      cd notebooks
+      echo Execute 03_develop_pipeline.ipynb
+      papermill 03_develop_pipeline.ipynb 03_develop_pipeline_output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '03_develop_pipeline.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      conda env list
+      cd notebooks  
+      echo Execute 04_deploy_logic_apps.ipynb
+      papermill 04_deploy_logic_apps.ipynb 04_deploy_logic_apps_output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '04_deploy_logic_apps.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      conda env list  
+      cd notebooks
+      echo Execute 05_clean_up.ipynb
+      papermill 05_clean_up.ipynb 05_clean_up_output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '05_clean_up.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate batchscoringdl_aml
+      echo Execute Resource Group Delete
+      existResponse=$(az group exists -n $(azurergname))
+      if [ "$existResponse" == "true" ]; then
+        echo Deleting project resource group  
+        az group delete --name $(azurergname) --yes
+      else
+        echo Project resource group did not exist
+      fi
+      echo Done Cleanup
+    displayName: 'Backup Cleanup'
+    condition: or(canceled(),failed())
+
+  - task: CreateWorkItem@1
+    inputs:
+      workItemType: 'Issue'
+      title: $(System.TeamProject) - Build $(Build.BuildNumber) Failed
+      assignedTo: 'JS <jiata@microsoft.com>'
+      associate: true
+      teamProject: $(System.TeamProject)
+    
+      fieldMappings: |
+        Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type.
+    displayName: 'Create work item on failure'
+    condition: failed()    
--- a/.ci/python-ml-scoring.yml
+++ b/.ci/python-ml-scoring.yml
@ -0,0 +1,71 @@
+# AMLBatchScoringPipeline Pipeline 
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+    
+variables:
+- group: AzureKeyVault
+
+jobs:
+- job: AMLBatchScoringPipelineJob
+  timeoutInMinutes: 300
+  cancelTimeoutInMinutes: 2
+  pool:
+    vmImage: 'Ubuntu-16.04'
+
+  steps:
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      which conda
+      conda env create -f environment.yml
+      conda env list
+      conda activate amlmm
+      conda env list  
+      echo Login Azure Account
+      az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret)
+    displayName: 'Configuration'
+  
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate amlmm
+      echo Execute 01_create_resources.ipynb
+      papermill 01_create_resources.ipynb 01_create_resources_Output.ipynb --log-output --no-progress-bar -k python3 -p SUBSCRIPTION_NAME  $(subscriptionname) -p RESOURCE_GROUP $(azurergname) -p LOCATION $(azurelocation)
+    displayName: '01_create_resources.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate amlmm
+      echo Execute 02_create_pipeline.ipynb
+      papermill 02_create_pipeline.ipynb 02_create_pipeline_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '02_create_pipeline.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate amlmm
+      echo Execute Resource Group Delete
+      existResponse=$(az group exists -n $(azurergname))
+      if [ "$existResponse" == "true" ]; then
+        echo Deleting resource group $(azurergname) 
+        az group delete --name $(azurergname) --yes
+      else
+        echo Resource group did not exist $(azurergname)
+      fi
+      echo Done Cleanup
+    displayName: 'Cleanup'
+    condition: always()
+
+  - task: CreateWorkItem@1
+    inputs:
+      workItemType: 'Issue'
+      title: $(System.TeamProject) - Build $(Build.BuildNumber) Failed
+      assignedTo: 'Katherine Lin <katlin@microsoft.com>'
+      associate: true
+      teamProject: $(System.TeamProject)
+    
+      fieldMappings: |
+        Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type.
+    displayName: 'Create work item on failure'
+    condition: failed()    
--- a/.ci/python-ml-training.yml
+++ b/.ci/python-ml-training.yml
@ -0,0 +1,118 @@
+# MLHyperparameterTuning Pipeline 
+
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+
+variables:
+- group: AzureKeyVault
+
+jobs:
+- job: MLHyperparameterTuningJob
+  timeoutInMinutes: 300
+  cancelTimeoutInMinutes: 2
+  pool:
+    vmImage: 'Ubuntu-16.04'
+
+  steps:
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      which conda
+      conda env create -f environment.yml
+      conda env list
+      conda activate MLHyperparameterTuning
+      conda env list  
+      echo Login Azure Account
+      az login -t $(sptenent) --service-principal -u $(spidentity) --password $(spsecret)
+      echo Try and figure out what account set takes
+      az account set -h
+      echo Try and set it.
+      az account set --subscription $(subscriptionid)
+#      papermill 01_Data_Prep.ipynb 01_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: 'Configuration'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 00_Data_Prep.ipynb
+      papermill 00_Data_Prep.ipynb 00_Data_Prep_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '00_Data_Prep.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 01_Training_Script.ipynb
+      papermill 01_Training_Script.ipynb 01_Training_Script_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '01_Training_Script.ipynb'
+ 
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 02_Testing_Script.ipynb
+      papermill 02_Testing_Script.ipynb 02_Testing_Script_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '02_Testing_Script.ipynb'
+ 
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 03_Run_Locally.ipynb
+      papermill 03_Run_Locally.ipynb 03_Run_Locally_Output.ipynb --log-output --no-progress-bar -k python3 -p selected_subscription $(subscriptionid) -p resource_group $(azurergname) 
+    displayName: '03_Run_Locally.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 04_Hyperparameter_Random_Search.ipynb
+      papermill 04_Hyperparameter_Random_Search.ipynb 04_Hyperparameter_Random_Search_Output.ipynb --log-output --no-progress-bar -k python3 -p max_total_runs $(dsmaxruns)
+    displayName: '04_Hyperparameter_Random_Search.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 05_Train_Best_Model.ipynb
+      papermill 05_Train_Best_Model.ipynb 05_Train_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '05_Train_Best_Model.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 06_Test_Best_Model.ipynb
+      papermill 06_Test_Best_Model.ipynb 06_Test_Best_Model_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '06_Test_Best_Model.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Executing 07_Tear_Down.ipynb
+      papermill 07_Tear_Down.ipynb 07_Tear_Down_Output.ipynb --log-output --no-progress-bar -k python3 
+    displayName: '07_Tear_Down.ipynb'
+
+  - bash: |
+      source /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate MLHyperparameterTuning
+      echo Execute Resource Group Delete
+      existResponse=$(az group exists -n $(azurergname))
+      if [ "$existResponse" == "true" ]; then
+        echo Deleting project resource group  
+        az group delete --name $(azurergname) --yes
+      else
+        echo Project resource group did not exist
+      fi
+      echo Done Cleanup
+    displayName: 'Backup Cleanup'
+    condition: or(canceled(),failed())
+
+  - task: CreateWorkItem@1
+    inputs:
+      workItemType: 'Issue'
+      title: $(System.TeamProject) - Build $(Build.BuildNumber) Failed
+      assignedTo: 'Mario Bourgoin <mabou@microsoft.com>'
+      associate: true
+      teamProject: $(System.TeamProject)
+    
+      fieldMappings: |
+        Description=Branch: Branch $(Build.SourceBranch) failed to build. Go to Boards>WorkItems and tag the failure type.
+    displayName: 'Create work item on failure'
+    condition: failed()