add schedule sample of CLI/SDK (#1524)
* add schedule sample of CLI/SDK * add ci workflows * temp update setup.sh to a tested wheel to test the workflow, need to remove this before merge back to main * update CLI setup.sh * update to a less frequent schedule * update samples to support doc * update sdk example for doc * update schedule sample * refine code to set import at begining and output list/get result * remove file prefix, add no-wait for async * clean up outputs * remove timezone in start time * update sdk version * update meta for doc * update cron expression * update yaml example * remove file prefix * update coments on start_time * remove file prefix * update for doc * update readme.py to auto generate schedule workflow and readme * add auto generated workflow and README for schedule * update script and workflow * minor readme table file name update * update readme file * remove the 2 workflows written manually * Fix format Signed-off-by: Brynn Yin <biyi@microsoft.com> * Add update part Signed-off-by: Brynn Yin <biyi@microsoft.com> * Remove unused import Signed-off-by: Brynn Yin <biyi@microsoft.com> * Refine enable/disable Signed-off-by: Brynn Yin <biyi@microsoft.com> * update schedule file names * update readme and workflow for cli * split runsettings Signed-off-by: Brynn Yin <biyi@microsoft.com> Co-authored-by: lochen <cloga0216@gmail.com> Co-authored-by: Brynn Yin <biyi@microsoft.com>
This commit is contained in:
eniac871
GitHub
Родитель
7eefa06793
Коммит
5dc65ced96
|
|
@ -0,0 +1,34 @@
|
|||
name: cli-schedules-schedules-cron-job-schedule
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0/4 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- sdk-preview
|
||||
paths:
|
||||
- cli/schedules/cron-job-schedule.yml
|
||||
- .github/workflows/cli-schedules-schedules-cron-job-schedule.yml
|
||||
- cli/setup.sh
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: check out repo
|
||||
uses: actions/checkout@v2
|
||||
- name: azure login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{secrets.AZ_CREDS}}
|
||||
- name: setup
|
||||
run: bash setup.sh
|
||||
working-directory: cli
|
||||
continue-on-error: true
|
||||
- name: create schedule
|
||||
run: az ml schedule create -f ./schedules/cron-job-schedule.yml --set name="ci_test_cron-job-schedule"
|
||||
working-directory: cli
|
||||
|
||||
- name: disable schedule
|
||||
run: az ml schedule disable --name ci_test_cron-job-schedule
|
||||
working-directory: cli
|
||||
34
.github/workflows/cli-schedules-schedules-cron-with-settings-job-schedule.yml
поставляемый
Normal file
34
.github/workflows/cli-schedules-schedules-cron-with-settings-job-schedule.yml
поставляемый
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
name: cli-schedules-schedules-cron-with-settings-job-schedule
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0/4 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- sdk-preview
|
||||
paths:
|
||||
- cli/schedules/cron-with-settings-job-schedule.yml
|
||||
- .github/workflows/cli-schedules-schedules-cron-with-settings-job-schedule.yml
|
||||
- cli/setup.sh
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: check out repo
|
||||
uses: actions/checkout@v2
|
||||
- name: azure login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{secrets.AZ_CREDS}}
|
||||
- name: setup
|
||||
run: bash setup.sh
|
||||
working-directory: cli
|
||||
continue-on-error: true
|
||||
- name: create schedule
|
||||
run: az ml schedule create -f ./schedules/cron-with-settings-job-schedule.yml --set name="ci_test_cron-with-settings-job-schedule"
|
||||
working-directory: cli
|
||||
|
||||
- name: disable schedule
|
||||
run: az ml schedule disable --name ci_test_cron-with-settings-job-schedule
|
||||
working-directory: cli
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
name: cli-schedules-schedules-recurrence-job-schedule
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0/4 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- sdk-preview
|
||||
paths:
|
||||
- cli/schedules/recurrence-job-schedule.yml
|
||||
- .github/workflows/cli-schedules-schedules-recurrence-job-schedule.yml
|
||||
- cli/setup.sh
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: check out repo
|
||||
uses: actions/checkout@v2
|
||||
- name: azure login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{secrets.AZ_CREDS}}
|
||||
- name: setup
|
||||
run: bash setup.sh
|
||||
working-directory: cli
|
||||
continue-on-error: true
|
||||
- name: create schedule
|
||||
run: az ml schedule create -f ./schedules/recurrence-job-schedule.yml --set name="ci_test_recurrence-job-schedule"
|
||||
working-directory: cli
|
||||
|
||||
- name: disable schedule
|
||||
run: az ml schedule disable --name ci_test_recurrence-job-schedule
|
||||
working-directory: cli
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
name: sdk-schedules-schedule
|
||||
# This file is created by sdk/readme.py.
|
||||
# Please do not edit directly.
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 */8 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- sdk/**
|
||||
- .github/workflows/sdk-schedules-job-schedule.yml
|
||||
- sdk/dev-requirements.txt
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: check out repo
|
||||
uses: actions/checkout@v2
|
||||
- name: setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: "3.8"
|
||||
- name: pip install notebook reqs
|
||||
run: pip install -r sdk/dev-requirements.txt
|
||||
- name: azure login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
creds: ${{secrets.AZ_CREDS}}
|
||||
- name: setup SDK
|
||||
run: bash setup.sh
|
||||
working-directory: sdk
|
||||
continue-on-error: true
|
||||
- name: setup CLI
|
||||
run: bash setup.sh
|
||||
working-directory: cli
|
||||
continue-on-error: true
|
||||
- name: run schedules/job-schedule.ipynb
|
||||
run: |
|
||||
sed -i -e "s/<SUBSCRIPTION_ID>/6560575d-fa06-4e7d-95fb-f962e74efd7a/g" job-schedule.ipynb
|
||||
sed -i -e "s/<RESOURCE_GROUP>/azureml-examples/g" job-schedule.ipynb
|
||||
sed -i -e "s/<AML_WORKSPACE_NAME>/main/g" job-schedule.ipynb
|
||||
sed -i -e "s/DefaultAzureCredential/AzureCliCredential/g" job-schedule.ipynb
|
||||
|
||||
papermill -k python job-schedule.ipynb job-schedule.output.ipynb
|
||||
working-directory: sdk/schedules
|
||||
- name: upload notebook's working folder as an artifact
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: schedule
|
||||
path: sdk/schedules
|
||||
|
|
@ -192,6 +192,14 @@ path|status|description
|
|||
[assets/model/local-file.yml](assets/model/local-file.yml)|[](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-model-local-file.yml)|Model created from local file.
|
||||
[assets/model/local-mlflow.yml](assets/model/local-mlflow.yml)|[](https://github.com/Azure/azureml-examples/actions/workflows/cli-assets-model-local-mlflow.yml)|Model created from local MLflow model directory.
|
||||
|
||||
**Schedules**
|
||||
|
||||
path|status|
|
||||
-|-
|
||||
[schedules/cron-job-schedule.yml](schedules/cron-job-schedule.yml)|[](https://github.com/Azure/azureml-examples/actions/workflows/cli-schedules-schedules/cron-job-schedule.yml)
|
||||
[schedules/cron-with-settings-job-schedule.yml](schedules/cron-with-settings-job-schedule.yml)|[](https://github.com/Azure/azureml-examples/actions/workflows/cli-schedules-schedules/cron-with-settings-job-schedule.yml)
|
||||
[schedules/recurrence-schedule.yml](schedules/recurrence-job-schedule.yml)|[](https://github.com/Azure/azureml-examples/actions/workflows/cli-schedules-schedules/recurrence-job-schedule.yml)
|
||||
|
||||
## Contents
|
||||
|
||||
|directory|description|
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ EXCLUDED_RESOURCES = [
|
|||
"compute/cluster-user-identity",
|
||||
]
|
||||
EXCLUDED_ASSETS = ["conda-yamls", "mlflow-models"]
|
||||
EXCLUDED_SCHEDULES = []
|
||||
EXCLUDED_SCRIPTS = ["setup", "cleanup", "run-job"]
|
||||
BRANCH = "main" # default - do not change
|
||||
# BRANCH = "sdk-preview" # this should be deleted when this branch is merged to main
|
||||
|
|
@ -81,15 +82,23 @@ def main(args):
|
|||
if not any(excluded in script for excluded in EXCLUDED_SCRIPTS)
|
||||
]
|
||||
|
||||
# get list of schedules
|
||||
schedules = sorted(glob.glob("schedules/**/*schedule.yml", recursive=True))
|
||||
schedules = [
|
||||
schedule.replace(".yml", "")
|
||||
for schedule in schedules
|
||||
if not any(excluded in schedule for excluded in EXCLUDED_SCHEDULES)
|
||||
]
|
||||
|
||||
# write workflows
|
||||
write_workflows(jobs, endpoints, resources, assets, scripts)
|
||||
write_workflows(jobs, endpoints, resources, assets, scripts, schedules)
|
||||
|
||||
# read existing README.md
|
||||
with open("README.md", "r") as f:
|
||||
readme_before = f.read()
|
||||
|
||||
# write README.md
|
||||
write_readme(jobs, endpoints, resources, assets, scripts)
|
||||
write_readme(jobs, endpoints, resources, assets, scripts, schedules)
|
||||
|
||||
# read modified README.md
|
||||
with open("README.md", "r") as f:
|
||||
|
|
@ -125,7 +134,7 @@ def modify_notebooks(notebooks):
|
|||
json.dump(data, f, indent=1)
|
||||
|
||||
|
||||
def write_readme(jobs, endpoints, resources, assets, scripts):
|
||||
def write_readme(jobs, endpoints, resources, assets, scripts, schedules):
|
||||
# read in prefix.md and suffix.md
|
||||
with open("prefix.md", "r") as f:
|
||||
prefix = f.read()
|
||||
|
|
@ -142,6 +151,7 @@ def write_readme(jobs, endpoints, resources, assets, scripts):
|
|||
)
|
||||
assets_table = "\n**Assets** ([assets](assets))\n\npath|status|description\n-|-|-\n"
|
||||
scripts_table = "\n**Scripts**\n\npath|status|\n-|-\n"
|
||||
schedules_table = "\n**Schedules**\n\npath|status|\n-|-\n"
|
||||
|
||||
# process jobs
|
||||
for job in jobs:
|
||||
|
|
@ -225,6 +235,16 @@ def write_readme(jobs, endpoints, resources, assets, scripts):
|
|||
row = f"[{script}.sh]({script}.sh)|{status}\n"
|
||||
scripts_table += row
|
||||
|
||||
# process schedules
|
||||
for schedule in schedules:
|
||||
# build entries for tutorial table
|
||||
status = f"[](https://github.com/Azure/azureml-examples/actions/workflows/cli-schedules-{schedule}.yml)"
|
||||
link = f"https://schedules.microsoft.com/azure/machine-learning/{schedule}"
|
||||
|
||||
# add row to tutorial table
|
||||
row = f"[{schedule}.yml]({schedule}.yml)|{status}\n"
|
||||
schedules_table += row
|
||||
|
||||
# write README.md
|
||||
print("writing README.md...")
|
||||
with open("README.md", "w") as f:
|
||||
|
|
@ -235,12 +255,13 @@ def write_readme(jobs, endpoints, resources, assets, scripts):
|
|||
+ endpoints_table
|
||||
+ resources_table
|
||||
+ assets_table
|
||||
+ schedules_table
|
||||
+ suffix
|
||||
)
|
||||
print("Finished writing README.md...")
|
||||
|
||||
|
||||
def write_workflows(jobs, endpoints, resources, assets, scripts):
|
||||
def write_workflows(jobs, endpoints, resources, assets, scripts, schedules):
|
||||
print("writing .github/workflows...")
|
||||
|
||||
# process jobs
|
||||
|
|
@ -269,6 +290,11 @@ def write_workflows(jobs, endpoints, resources, assets, scripts):
|
|||
# write workflow file
|
||||
write_script_workflow(script)
|
||||
|
||||
# process schedules
|
||||
for schedule in schedules:
|
||||
# write workflow file
|
||||
write_schedule_workflow(schedule)
|
||||
|
||||
|
||||
def check_readme(before, after):
|
||||
return before == after
|
||||
|
|
@ -453,6 +479,48 @@ jobs:
|
|||
f.write(workflow_yaml)
|
||||
|
||||
|
||||
def write_schedule_workflow(schedule):
|
||||
filename, project_dir, hyphenated = parse_path(schedule)
|
||||
creds = "${{secrets.AZ_CREDS}}"
|
||||
workflow_yaml = f"""name: cli-schedules-{hyphenated}
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 0/4 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- sdk-preview
|
||||
paths:
|
||||
- cli/{schedule}.yml
|
||||
- .github/workflows/cli-schedules-{hyphenated}.yml
|
||||
- cli/setup.sh
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: check out repo
|
||||
uses: actions/checkout@v2
|
||||
- name: azure login
|
||||
uses: azure/login@v1
|
||||
with:
|
||||
creds: {creds}
|
||||
- name: setup
|
||||
run: bash setup.sh
|
||||
working-directory: cli
|
||||
continue-on-error: true
|
||||
- name: create schedule
|
||||
run: az ml schedule create -f ./{schedule}.yml --set name="ci_test_{filename}"
|
||||
working-directory: cli\n
|
||||
- name: disable schedule
|
||||
run: az ml schedule disable --name ci_test_{filename}
|
||||
working-directory: cli\n"""
|
||||
|
||||
# write workflow
|
||||
with open(f"../.github/workflows/cli-schedules-{hyphenated}.yml", "w") as f:
|
||||
f.write(workflow_yaml)
|
||||
|
||||
|
||||
# run functions
|
||||
if __name__ == "__main__":
|
||||
# issue #146
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
## Working with Schedule in Azure Machine Learning CLI 2.0
|
||||
This repository contains an example `YAML` file for creating `schedule` using Azure Machine learning CLI 2.0. This directory includes:
|
||||
|
||||
- Sample `YAML` files for creating a `schedule`. Currently the `schedule` can be used to associate with a `PipelineJob`, more job types & other resources will be supported in near future.
|
||||
|
||||
|
||||
- To create a schedule using any of the sample YAML files end with "-schedule.yml" provided in this directory, execute following command:
|
||||
```cli
|
||||
> az ml schedule create -f cron-job-schedule.yml
|
||||
```
|
||||
|
||||
- To list all the schedule from your workspace, execute following command:
|
||||
```cli
|
||||
> az ml schedule list -o table
|
||||
```
|
||||
|
||||
- To quickly enable/disable an existing schedule, execute following command:
|
||||
```cli
|
||||
> az ml schedule enable -n schedule_name
|
||||
> az ml schedule disable -n schedule_name
|
||||
```
|
||||
|
||||
- To delete a schedule, execute following command, after delete a schedule, the schedule cannot trigger jobs any more, and cannot be recovered:
|
||||
```cli
|
||||
> az ml schedule delete -n schedule_name
|
||||
```
|
||||
|
||||
- To update a schedule that in workspace, execute following command. Currently schedule expression, job, settings and input/output are support update:
|
||||
```cli
|
||||
> az ml schedule update -f cron-job-schedule.yml --set trigger.expression="15 * * * *"
|
||||
```
|
||||
|
||||
- Customer can create multiple schedules associate with same job. To list all the schedules associate with the job, execute following command:
|
||||
```cli
|
||||
> az ml schedule list --job_name job_name
|
||||
```
|
||||
|
||||
To learn more about Azure Machine Learning CLI 2.0, [follow this link](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-configure-cli).
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
$schema: https://azuremlschemas.azureedge.net/latest/schedule.schema.json
|
||||
name: simple_cron_job_schedule
|
||||
display_name: Simple cron job schedule
|
||||
description: a simple hourly cron job schedule
|
||||
|
||||
trigger:
|
||||
type: cron
|
||||
expression: "0 * * * *"
|
||||
start_time: "2022-07-10T10:00:00" # optional - default will be schedule creation time
|
||||
time_zone: "Pacific Standard Time" # optional - default will be UTC
|
||||
|
||||
# create_job: azureml:simple-pipeline-job
|
||||
create_job: ./simple-pipeline-job.yml
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
$schema: https://azuremlschemas.azureedge.net/latest/schedule.schema.json
|
||||
name: cron_with_settings_job_schedule
|
||||
display_name: Simple cron job schedule
|
||||
description: a simple hourly cron job schedule
|
||||
|
||||
trigger:
|
||||
type: cron
|
||||
expression: "0 * * * *"
|
||||
start_time: "2022-07-10T10:00:00" # optional - default will be schedule creation time
|
||||
time_zone: "Pacific Standard Time" # optional - default will be UTC
|
||||
|
||||
create_job:
|
||||
type: pipeline
|
||||
job: ./simple-pipeline-job.yml
|
||||
# job: azureml:simple-pipeline-job
|
||||
# runtime settings
|
||||
settings:
|
||||
#default_compute: azureml:cpu-cluster
|
||||
continue_on_step_failure: true
|
||||
inputs:
|
||||
hello_string_top_level_input: ${{name}}
|
||||
tags:
|
||||
schedule: cron_with_settings_schedule
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
$schema: https://azuremlschemas.azureedge.net/latest/schedule.schema.json
|
||||
name: simple_recurrence_job_schedule
|
||||
display_name: Simple recurrence job schedule
|
||||
description: a simple hourly recurrence job schedule
|
||||
|
||||
trigger:
|
||||
type: recurrence
|
||||
frequency: day #can be minute, hour, day, week, month
|
||||
interval: 1 #every day
|
||||
schedule:
|
||||
hours: [4,5,10,11,12]
|
||||
minutes: [0,30]
|
||||
start_time: "2022-07-10T10:00:00" # optional - default will be schedule creation time
|
||||
time_zone: "Pacific Standard Time" # optional - default will be UTC
|
||||
|
||||
create_job: ./simple-pipeline-job.yml
|
||||
# create_job: azureml:simple-pipeline-job
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
# <create_schedule>
|
||||
# This action will create related resources for a schedule. It will take dozens of seconds to complete.
|
||||
az ml schedule create --file cron-schedule.yml --no-wait
|
||||
# </create_schedule>
|
||||
|
||||
# <show_schedule>
|
||||
az ml schedule show -n simple_cron_schedule
|
||||
# </show_schedule>
|
||||
|
||||
# <list_schedule>
|
||||
az ml schedule list
|
||||
# </list_schedule>
|
||||
|
||||
# <update_schedule>
|
||||
az ml schedule update -n simple_cron_schedule --set description="new description" --no-wait
|
||||
# </update_schedule>
|
||||
|
||||
# <disable_schedule>
|
||||
az ml schedule disable -n simple_cron_schedule --no-wait
|
||||
# </disable_schedule>
|
||||
|
||||
# <enable_schedule>
|
||||
az ml schedule enable -n simple_cron_schedule --no-wait
|
||||
# </enable_schedule>
|
||||
|
||||
# <query_triggered_jobs>
|
||||
# query triggered jobs from schedule, please replace the simple_cron_schedule to your schedule name
|
||||
az ml job list --query "[?contains(display_name,'simple_cron_schedule')]"
|
||||
# </query_triggered_jobs>
|
||||
|
||||
# <delete_schedule>
|
||||
az ml schedule delete -n simple_cron_schedule
|
||||
# </delete_schedule>
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
$schema: https://azuremlschemas.azureedge.net/latest/pipelineJob.schema.json
|
||||
type: pipeline
|
||||
display_name: hello_pipeline_abc
|
||||
compute: azureml:cpu-cluster
|
||||
|
||||
inputs:
|
||||
hello_string_top_level_input: "hello world"
|
||||
jobs:
|
||||
a:
|
||||
command: echo hello ${{inputs.hello_string}}
|
||||
environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu@latest
|
||||
inputs:
|
||||
hello_string: ${{parent.inputs.hello_string_top_level_input}}
|
||||
b:
|
||||
command: echo "world" >> ${{outputs.world_output}}/world.txt
|
||||
environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu@latest
|
||||
outputs:
|
||||
world_output:
|
||||
c:
|
||||
command: echo ${{inputs.world_input}}/world.txt
|
||||
environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu@latest
|
||||
inputs:
|
||||
world_input: ${{parent.jobs.b.outputs.world_output}}
|
||||
|
||||
|
|
@ -16,4 +16,4 @@ WORKSPACE="main"
|
|||
|
||||
# <az_configure_defaults>
|
||||
az configure --defaults group=$GROUP workspace=$WORKSPACE location=$LOCATION
|
||||
# </az_configure_defaults>
|
||||
# </az_configure_defaults>
|
||||
|
|
@ -0,0 +1,412 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Working with Schedule\n",
|
||||
"\n",
|
||||
"**Requirements** - In order to benefit from this tutorial, you will need:\n",
|
||||
"- A basic understanding of Machine Learning\n",
|
||||
"- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n",
|
||||
"- An Azure ML workspace - [Configure workspace](../../jobs/configuration.ipynb) \n",
|
||||
"- A python environment\n",
|
||||
"- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../README.md) - check the getting started section\n",
|
||||
"\n",
|
||||
"**Learning Objectives** - By the end of this tutorial, you should be able to:\n",
|
||||
"- Create schedule from Python SDK\n",
|
||||
"- Enable/Disable schedule from Python SDK\n",
|
||||
"- Update schedule's trigger from Python SDK\n",
|
||||
"\n",
|
||||
"**Motivations** - This notebook covers the main scenario that user management schedule for job in SDK v2."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 1. Connect to Azure Machine Learning Workspace\n",
|
||||
"\n",
|
||||
"The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n",
|
||||
"\n",
|
||||
"## 1.1. Import the required libraries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import required libraries\n",
|
||||
"from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n",
|
||||
"\n",
|
||||
"from azure.ai.ml import MLClient, Input, load_component\n",
|
||||
"from azure.ai.ml.constants import TimeZone\n",
|
||||
"from azure.ai.ml.dsl import pipeline\n",
|
||||
"from azure.ai.ml.entities import (\n",
|
||||
" JobSchedule,\n",
|
||||
" CronTrigger,\n",
|
||||
" RecurrenceTrigger,\n",
|
||||
" RecurrencePattern,\n",
|
||||
")\n",
|
||||
"from datetime import datetime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1.2. Configure credential\n",
|
||||
"\n",
|
||||
"We are using `DefaultAzureCredential` to get access to workspace. When an access token is needed, it requests one using multiple identities(`EnvironmentCredential, ManagedIdentityCredential, SharedTokenCacheCredential, VisualStudioCodeCredential, AzureCliCredential, AzurePowerShellCredential`) in turn, stopping when one provides a token.\n",
|
||||
"Reference [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for more information.\n",
|
||||
"\n",
|
||||
"`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n",
|
||||
"Reference [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python) for all available credentials if it does not work for you. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" credential = DefaultAzureCredential()\n",
|
||||
" # Check if given credential can get token successfully.\n",
|
||||
" credential.get_token(\"https://management.azure.com/.default\")\n",
|
||||
"except Exception as ex:\n",
|
||||
" # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n",
|
||||
" credential = InteractiveBrowserCredential()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1.3. Configure workspace details and get a handle to the workspace\n",
|
||||
"\n",
|
||||
"To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. We will use these details in the `MLClient` from `azure.ai.ml` to get a handle to the required Azure Machine Learning workspace."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"try:\n",
|
||||
" ml_client = MLClient.from_config(credential=credential)\n",
|
||||
"except Exception as ex:\n",
|
||||
" # enter details of your AML workspace\n",
|
||||
" subscription_id = \"<SUBSCRIPTION_ID>\"\n",
|
||||
" resource_group = \"<RESOURCE_GROUP>\"\n",
|
||||
" workspace = \"<AML_WORKSPACE_NAME>\"\n",
|
||||
"\n",
|
||||
" # get a handle to the workspace\n",
|
||||
" ml_client = MLClient(credential, subscription_id, resource_group, workspace)\n",
|
||||
"print(ml_client)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1.4 Create a pipeline job with settings"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "create_pipeline_job"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"parent_dir = \"../jobs/pipelines/1a_pipeline_with_components_from_yaml\"\n",
|
||||
"train_model = load_component(path=parent_dir + \"/train_model.yml\")\n",
|
||||
"score_data = load_component(path=parent_dir + \"/score_data.yml\")\n",
|
||||
"eval_model = load_component(path=parent_dir + \"/eval_model.yml\")\n",
|
||||
"\n",
|
||||
"# Construct pipeline\n",
|
||||
"@pipeline()\n",
|
||||
"def pipeline_with_components_from_yaml(\n",
|
||||
" training_input,\n",
|
||||
" test_input,\n",
|
||||
" training_max_epochs=20,\n",
|
||||
" training_learning_rate=1.8,\n",
|
||||
" learning_rate_schedule=\"time-based\",\n",
|
||||
"):\n",
|
||||
" \"\"\"E2E dummy train-score-eval pipeline with components defined via yaml.\"\"\"\n",
|
||||
" # Call component obj as function: apply given inputs & parameters to create a node in pipeline\n",
|
||||
" train_with_sample_data = train_model(\n",
|
||||
" training_data=training_input,\n",
|
||||
" max_epochs=training_max_epochs,\n",
|
||||
" learning_rate=training_learning_rate,\n",
|
||||
" learning_rate_schedule=learning_rate_schedule,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" score_with_sample_data = score_data(\n",
|
||||
" model_input=train_with_sample_data.outputs.model_output, test_data=test_input\n",
|
||||
" )\n",
|
||||
" score_with_sample_data.outputs.score_output.mode = \"upload\"\n",
|
||||
"\n",
|
||||
" eval_with_sample_data = eval_model(\n",
|
||||
" scoring_result=score_with_sample_data.outputs.score_output\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Return: pipeline outputs\n",
|
||||
" return {\n",
|
||||
" \"trained_model\": train_with_sample_data.outputs.model_output,\n",
|
||||
" \"scored_data\": score_with_sample_data.outputs.score_output,\n",
|
||||
" \"evaluation_report\": eval_with_sample_data.outputs.eval_output,\n",
|
||||
" }"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "change_run_settings"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# set run time settings\n",
|
||||
"pipeline_job = pipeline_with_components_from_yaml(\n",
|
||||
" training_input=Input(type=\"uri_folder\", path=parent_dir + \"/data/\"),\n",
|
||||
" test_input=Input(type=\"uri_folder\", path=parent_dir + \"/data/\"),\n",
|
||||
" training_max_epochs=20,\n",
|
||||
" training_learning_rate=1.8,\n",
|
||||
" learning_rate_schedule=\"time-based\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# set pipeline level compute\n",
|
||||
"pipeline_job.settings.default_compute = \"cpu-cluster\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.1 Create schedule\n",
|
||||
"### 2.1.1 Define schedule with with recurrence pattern"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "create_schedule_recurrence"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"schedule_name = \"simple_sdk_create_schedule_recurrence\"\n",
|
||||
"\n",
|
||||
"schedule_start_time = datetime.now()\n",
|
||||
"recurrence_trigger = RecurrenceTrigger(\n",
|
||||
" frequency=\"day\",\n",
|
||||
" interval=1,\n",
|
||||
" schedule=RecurrencePattern(hours=10, minutes=[0, 1]),\n",
|
||||
" start_time=schedule_start_time,\n",
|
||||
" time_zone=TimeZone.UTC,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"job_schedule = JobSchedule(\n",
|
||||
" name=schedule_name, trigger=recurrence_trigger, create_job=pipeline_job\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.1.2 Define schedule with with cron expression"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "create_schedule_cron"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"schedule_name = \"simple_sdk_create_schedule_cron\"\n",
|
||||
"\n",
|
||||
"cron_trigger = CronTrigger(\n",
|
||||
" expression=\"15 10 * * *\",\n",
|
||||
" start_time=\"2022-07-10T10:00:00\", # start time\n",
|
||||
" time_zone=\"Eastern Standard Time\", # time zone of expression\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"job_schedule = JobSchedule(\n",
|
||||
" name=schedule_name, trigger=cron_trigger, create_job=pipeline_job\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.1.3 Create schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "create_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job_schedule = ml_client.schedules.begin_create_or_update(schedule=job_schedule)\n",
|
||||
"print(job_schedule)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.2 Disable the schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "disable_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job_schedule = ml_client.schedules.begin_disable(name=schedule_name)\n",
|
||||
"job_schedule.is_enabled"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.3 Check the detail of the schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "show_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"created_schedule = ml_client.schedules.get(name=schedule_name)\n",
|
||||
"[created_schedule.name]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.4 List schedules in a workspace"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "list_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"schedules = ml_client.schedules.list()\n",
|
||||
"[s.name for s in schedules]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.5 Enable a schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job_schedule = ml_client.schedules.begin_enable(name=schedule_name)\n",
|
||||
"job_schedule.is_enabled"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.6 Update a schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "enable_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Update trigger expression\n",
|
||||
"job_schedule.trigger.expression = \"10 10 * * 1\"\n",
|
||||
"job_schedule = ml_client.schedules.begin_create_or_update(schedule=job_schedule)\n",
|
||||
"print(job_schedule)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.7 Delete the schedule"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"name": "delete_schedule"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Only disabled schedules can be deleted\n",
|
||||
"ml_client.schedules.begin_disable(name=schedule_name)\n",
|
||||
"ml_client.schedules.begin_delete(name=schedule_name)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"description": {
|
||||
"description": "Create a component asset"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.7.8 64-bit",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.8"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "727b410030f20083a8dadbe78e3f0c3cba212580ac111eb7b243827f2188ccac"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче