remove old pipeline
This commit is contained in:
j-so
Родитель
dbe8b33ab2
Коммит
2796b40024
|
|
@ -30,14 +30,9 @@ variables:
|
|||
value: 'scoring/scoreB.py'
|
||||
|
||||
steps:
|
||||
- task: AzureCLI@1
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
inlineScript: |
|
||||
set -e
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python3 -m ml_service.util.create_scoring_image
|
||||
displayName: 'Create Scoring Image'
|
||||
- template: diabetes_regression-package-model-template.yml
|
||||
parameters:
|
||||
modelId: $(MODEL_NAME):$(MODEL_VERSION)
|
||||
scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/$(SCORE_SCRIPT)'
|
||||
condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
|
||||
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model.
|
||||
|
||||
resources:
|
||||
containers:
|
||||
- container: mlops
|
||||
image: mcr.microsoft.com/mlops/python:latest
|
||||
|
||||
pr: none
|
||||
trigger:
|
||||
branches:
|
||||
include:
|
||||
- master
|
||||
paths:
|
||||
include:
|
||||
- diabetes_regression/
|
||||
- ml_service/pipelines/diabetes_regression_build_train_pipeline.py
|
||||
- ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py
|
||||
- ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py
|
||||
|
||||
variables:
|
||||
- template: diabetes_regression-variables-template.yml
|
||||
- group: devopsforai-aml-vg
|
||||
|
||||
pool:
|
||||
vmImage: ubuntu-latest
|
||||
|
||||
stages:
|
||||
- stage: 'Model_CI'
|
||||
displayName: 'Model CI'
|
||||
jobs:
|
||||
- job: "Model_CI_Pipeline"
|
||||
displayName: "Model CI Pipeline"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: code-quality-template.yml
|
||||
- task: AzureCLI@1
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
# Invoke the Python building and publishing a training pipeline
|
||||
python -m ml_service.pipelines.diabetes_regression_build_train_pipeline
|
||||
displayName: 'Publish Azure Machine Learning Pipeline'
|
||||
|
||||
- stage: 'Trigger_AML_Pipeline'
|
||||
displayName: 'Train and evaluate model'
|
||||
condition: succeeded()
|
||||
variables:
|
||||
BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)'
|
||||
jobs:
|
||||
- job: "Get_Pipeline_ID"
|
||||
condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true'))
|
||||
displayName: "Get Pipeline ID for execution"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- task: AzureCLI@1
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution
|
||||
# Set AMLPIPELINEID variable for next AML Pipeline task in next job
|
||||
AMLPIPELINEID="$(cat pipeline_id.txt)"
|
||||
echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID"
|
||||
name: 'getpipelineid'
|
||||
displayName: 'Get Pipeline ID'
|
||||
- job: "Run_ML_Pipeline"
|
||||
dependsOn: "Get_Pipeline_ID"
|
||||
displayName: "Trigger ML Training Pipeline"
|
||||
timeoutInMinutes: 0
|
||||
pool: server
|
||||
variables:
|
||||
AMLPIPELINE_ID: $[ dependencies.Get_Pipeline_ID.outputs['getpipelineid.AMLPIPELINEID'] ]
|
||||
steps:
|
||||
- task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0
|
||||
displayName: 'Invoke ML pipeline'
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
PipelineId: '$(AMLPIPELINE_ID)'
|
||||
ExperimentName: '$(EXPERIMENT_NAME)'
|
||||
PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)"}, "tags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}, "StepTags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}'
|
||||
- job: "Training_Run_Report"
|
||||
dependsOn: "Run_ML_Pipeline"
|
||||
condition: always()
|
||||
displayName: "Publish artifact if new model was registered"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: diabetes_regression-publish-model-artifact-template.yml
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model.
|
||||
# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model.
|
||||
|
||||
resources:
|
||||
containers:
|
||||
|
|
@ -27,7 +27,6 @@ pool:
|
|||
stages:
|
||||
- stage: 'Model_CI'
|
||||
displayName: 'Model CI'
|
||||
condition: not(variables['MODEL_BUILD_ID'])
|
||||
jobs:
|
||||
- job: "Model_CI_Pipeline"
|
||||
displayName: "Model CI Pipeline"
|
||||
|
|
@ -48,8 +47,8 @@ stages:
|
|||
displayName: 'Publish Azure Machine Learning Pipeline'
|
||||
|
||||
- stage: 'Trigger_AML_Pipeline'
|
||||
displayName: 'Train model'
|
||||
condition: and(succeeded(), not(variables['MODEL_BUILD_ID']))
|
||||
displayName: 'Train and evaluate model'
|
||||
condition: succeeded()
|
||||
variables:
|
||||
BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)'
|
||||
jobs:
|
||||
|
|
@ -91,116 +90,8 @@ stages:
|
|||
- job: "Training_Run_Report"
|
||||
dependsOn: "Run_ML_Pipeline"
|
||||
condition: always()
|
||||
displayName: "Determine if evaluation succeeded and new model is registered"
|
||||
displayName: "Publish artifact if new model was registered"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: diabetes_regression-get-model-version-template.yml
|
||||
|
||||
- stage: 'Deploy_ACI'
|
||||
displayName: 'Deploy to ACI'
|
||||
dependsOn: Trigger_AML_Pipeline
|
||||
condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['ACI_DEPLOYMENT_NAME'])
|
||||
jobs:
|
||||
- job: "Deploy_ACI"
|
||||
displayName: "Deploy to ACI"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: diabetes_regression-get-model-version-template.yml
|
||||
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
|
||||
displayName: 'Azure ML Model Deploy'
|
||||
inputs:
|
||||
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
|
||||
modelSourceType: manualSpec
|
||||
modelName: '$(MODEL_NAME)'
|
||||
modelVersion: $(MODEL_VERSION)
|
||||
inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
|
||||
deploymentTarget: ACI
|
||||
deploymentName: $(ACI_DEPLOYMENT_NAME)
|
||||
deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml'
|
||||
overwriteExistingDeployment: true
|
||||
- task: AzureCLI@1
|
||||
displayName: 'Smoke test'
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
|
||||
|
||||
- stage: 'Deploy_AKS'
|
||||
displayName: 'Deploy to AKS'
|
||||
dependsOn: Deploy_ACI
|
||||
condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
|
||||
jobs:
|
||||
- job: "Deploy_AKS"
|
||||
displayName: "Deploy to AKS"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: diabetes_regression-get-model-version-template.yml
|
||||
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
|
||||
displayName: 'Azure ML Model Deploy'
|
||||
inputs:
|
||||
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
|
||||
modelSourceType: manualSpec
|
||||
modelName: '$(MODEL_NAME)'
|
||||
modelVersion: $(MODEL_VERSION)
|
||||
inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
|
||||
deploymentTarget: AKS
|
||||
aksCluster: $(AKS_COMPUTE_NAME)
|
||||
deploymentName: $(AKS_DEPLOYMENT_NAME)
|
||||
deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml'
|
||||
overwriteExistingDeployment: true
|
||||
- task: AzureCLI@1
|
||||
displayName: 'Smoke test'
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
|
||||
|
||||
- stage: 'Deploy_Webapp'
|
||||
displayName: 'Deploy to Webapp'
|
||||
dependsOn: Trigger_AML_Pipeline
|
||||
condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['WEBAPP_DEPLOYMENT_NAME'])
|
||||
jobs:
|
||||
- job: "Deploy_Webapp"
|
||||
displayName: "Deploy to Webapp"
|
||||
container: mlops
|
||||
timeoutInMinutes: 0
|
||||
steps:
|
||||
- template: diabetes_regression-get-model-version-template.yml
|
||||
- task: AzureCLI@1
|
||||
displayName: 'Create scoring image and set IMAGE_LOCATION variable'
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
|
||||
# Output image location to Azure DevOps job
|
||||
IMAGE_LOCATION="$(cat image_location.txt)"
|
||||
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
|
||||
- task: AzureWebAppContainer@1
|
||||
name: WebAppDeploy
|
||||
displayName: 'Azure Web App on Container Deploy'
|
||||
inputs:
|
||||
azureSubscription: '$(AZURE_RM_SVC_CONNECTION)'
|
||||
appName: '$(WEBAPP_DEPLOYMENT_NAME)'
|
||||
resourceGroupName: '$(RESOURCE_GROUP)'
|
||||
imageName: '$(IMAGE_LOCATION)'
|
||||
- task: AzureCLI@1
|
||||
displayName: 'Smoke test'
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
|
||||
- template: diabetes_regression-publish-model-artifact-template.yml
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
# Pipeline template that attempts to get the latest model version and adds it to the environment for subsequent tasks to use.
|
||||
steps:
|
||||
- task: AzureCLI@1
|
||||
inputs:
|
||||
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
|
||||
scriptLocation: inlineScript
|
||||
inlineScript: |
|
||||
set -e # fail on error
|
||||
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
|
||||
python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(modelbuildid) --output_model_version_file "model_version.txt"
|
||||
# Output model version to Azure DevOps job
|
||||
MODEL_VERSION="$(cat model_version.txt)"
|
||||
echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"
|
||||
name: 'getversion'
|
||||
displayName: "Determine if evaluation succeeded and new model is registered"
|
||||
|
|
@ -127,25 +127,30 @@ Create a new service connection to your Azure ML Workspace using the [Machine Le
|
|||
**Note:** Similar to the Azure Resource Manager service connection you created earlier, creating a service connection with Azure Machine Learning workspace scope requires 'Owner' or 'User Access Administrator' permissions on the Workspace.
|
||||
You'll need sufficient permissions to register an application with your Azure AD tenant, or you can get the ID and secret of a service principal from your Azure AD Administrator. That principal must have Contributor permissions on the Azure ML Workspace.
|
||||
|
||||
## Set up Build, Release Trigger, and Release Multi-Stage Pipeline
|
||||
## Set up Build, Release Trigger, and Release Multi-Stage Pipelines
|
||||
|
||||
Now that you've provisioned all the required Azure resources and service connections, you can set up the pipeline for deploying your machine learning model to production. The pipeline has a sequence of stages for:
|
||||
Now that you've provisioned all the required Azure resources and service connections, you can set up the pipelines for training (CI) and deploying (CD) your machine learning model to production.
|
||||
|
||||
1. **Model Code Continuous Integration:** triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage and publishes a training pipeline.
|
||||
1. **Train Model**: invokes the Azure ML service to trigger the published training pipeline to train, evaluate, and register a model.
|
||||
1. **Release Deployment:** deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. For simplicity, you're going to initially focus on Azure Container Instances. See [Further Exploration](#further-exploration) for other deployment types.
|
||||
1. **Model CI, training, evaluation, and registration** - triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage, and publishes and runs the training pipeline. If a new model is registered after evaluation, it creates a build artifact containing the JSON metadata of the model. Definition: [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml).
|
||||
1. **Release deployment** - consumes the artifact of the previous pipeline and deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. See [Further Exploration](#further-exploration) for other deployment types. Definition: [diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml).
|
||||
1. **Note:** Edit the pipeline definition to remove unused stages. For example, if you're deploying to Azure Container Instances and Azure Kubernetes Service only, delete the unused `Deploy_Webapp` stage.
|
||||
|
||||
### Set up the Pipeline
|
||||
These pipelines use a Docker container on the Azure Pipelines agents to accomplish the pipeline steps. The container image ***mcr.microsoft.com/mlops/python:latest*** is built with [this Dockerfile](../environment_setup/Dockerfile) and has all the necessary dependencies installed for MLOpsPython and ***diabetes_regression***. This image is an example of a custom Docker image with a pre-baked environment. The environment is guaranteed to be the same on any building agent, VM, or local machine. In your project, you'll want to build your own Docker image that only contains the dependencies and tools required for your use case. Your image will probably be smaller and faster, and it will be maintained by your team.
|
||||
|
||||
### Set up the Model CI, training, evaluation, and registration pipeline
|
||||
|
||||
In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml)
|
||||
pipeline definition in your forked repository.
|
||||
|
||||
|
||||
If you plan to use the release deployment pipeline (in the next section), you will need to rename this pipeline to `Model-Train-Register-CI`.
|
||||
|
||||
Once the pipeline is finished, check the execution result:
|
||||
|
||||
|
||||
|
||||
|
||||
And the pipeline artifacts:
|
||||
|
||||
|
||||
|
||||
Also check the published training pipeline in the **mlops-AML-WS** workspace in [Azure Portal](https://portal.azure.com/):
|
||||
|
||||
|
|
@ -153,6 +158,12 @@ Also check the published training pipeline in the **mlops-AML-WS** workspace in
|
|||
|
||||
Great, you now have the build pipeline set up which automatically triggers every time there's a change in the master branch!
|
||||
|
||||
After the pipeline is finished, you'll see a new model in the **ML Workspace**:
|
||||
|
||||
|
||||
|
||||
To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. You can also override the variable at runtime execution of the pipeline.
|
||||
|
||||
The pipeline stages are summarized below:
|
||||
|
||||
#### Model CI
|
||||
|
|
@ -168,28 +179,64 @@ The pipeline stages are summarized below:
|
|||
- This is an **agentless** job. The CI pipeline can wait for ML pipeline completion for hours or even days without using agent resources.
|
||||
- Determine if a new model was registered by the *ML Training Pipeline*.
|
||||
- If the model evaluation determines that the new model doesn't perform any better than the previous one, the new model won't register and the *ML Training Pipeline* will be **canceled**. In this case, you'll see a message in the 'Train Model' job under the 'Determine if evaluation succeeded and new model is registered' step saying '**Model was not registered for this run.**'
|
||||
- See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic and [diabetes_regression_verify_train_pipeline.py](../ml_service/pipelines/diabetes_regression_verify_train_pipeline.py#L54) for the ML pipeline reporting logic.
|
||||
- See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic.
|
||||
- [Additional Variables and Configuration](#additional-variables-and-configuration) for configuring this and other behavior.
|
||||
|
||||
#### Create pipeline artifact
|
||||
|
||||
- Get the info about the registered model
|
||||
- Create a pipeline artifact called `model` that contains a `model.json` file containing the model information.
|
||||
|
||||
### Set up the Release deployment pipeline
|
||||
|
||||
---
|
||||
**PREREQUISITE**
|
||||
|
||||
In order to use this pipeline:
|
||||
|
||||
1. Follow the steps to set up the Model CI, training, evaluation, and registration pipeline.
|
||||
1. You **must** rename your model CI/train/eval/register pipeline to `Model-Train-Register-CI`.
|
||||
|
||||
The release deploymment pipeline relies on the model CI pipeline and references it by name.
|
||||
|
||||
---
|
||||
|
||||
In this section, we will set up the pipeline for release deployment to ACI, AKS, or Webapp. This pipeline uses the resulting artifact of the [Model-Train-Register-CI pipeline](#) to identify the model to be deployed.
|
||||
|
||||
This pipeline has the following behaviors:
|
||||
|
||||
- The pipeline will **automatically trigger** on completion of the Model-Train-Register-CI pipeline
|
||||
- The pipeline will default to using the latest successful build of the Model-Train-Register-CI pipeline. It will deploy the model produced by that build.
|
||||
- You can specify a `Model-Train-Register-CI` build ID when running the pipeline manually. You can find this in the url of the build, and the model registered from that build will also be tagged with the build ID. This is useful to skip model training and registration, and deploy a model successfully registered by a `Model-Train-Register-CI` build.
|
||||
|
||||
In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml)
|
||||
pipeline definition in your forked repository.
|
||||
|
||||
Your first run will use the latest model created by the `Model-Train-Register-CI` pipeline.
|
||||
|
||||
Once the pipeline is finished, check the execution result:
|
||||
|
||||
|
||||
|
||||
To specify a particular build's model, set the `Model Train CI Build Id` parameter to the build Id you would like to use.
|
||||
|
||||
|
||||
|
||||
Once your pipeline run begins, you can see the model name and version downloaded from the `Model-Train-Register-CI` pipeline.
|
||||
|
||||
|
||||
|
||||
The pipeline has the following stage:
|
||||
|
||||
#### Deploy to ACI
|
||||
|
||||
- Deploy the model to the QA environment in [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/).
|
||||
- Smoke test
|
||||
- The test sends a sample query to the scoring web service and verifies that it returns the expected response. Have a look at the [smoke test code](../ml_service/util/smoke_test_scoring_service.py) for an example.
|
||||
|
||||
The pipeline uses a Docker container on the Azure Pipelines agents to accomplish the pipeline steps. The container image ***mcr.microsoft.com/mlops/python:latest*** is built with [this Dockerfile](../environment_setup/Dockerfile) and has all the necessary dependencies installed for MLOpsPython and ***diabetes_regression***. This image is an example of a custom Docker image with a pre-baked environment. The environment is guaranteed to be the same on any building agent, VM, or local machine. In your project, you'll want to build your own Docker image that only contains the dependencies and tools required for your use case. Your image will probably be smaller and faster, and it will be maintained by your team.
|
||||
|
||||
After the pipeline is finished, you'll see a new model in the **ML Workspace**:
|
||||
|
||||
|
||||
|
||||
To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. You can also override the variable at runtime execution of the pipeline.
|
||||
|
||||
To skip model training and registration, and deploy a model successfully registered by a previous build (for testing changes to the score file or inference configuration), add the variable `MODEL_BUILD_ID` when the pipeline is queued, and set the value to the ID of the previous build.
|
||||
|
||||
## Further Exploration
|
||||
|
||||
You should now have a working pipeline that can get you started with MLOpsPython. Below are some additional features offered that might suit your scenario.
|
||||
You should now have a working set of pipelines that can get you started with MLOpsPython. Below are some additional features offered that might suit your scenario.
|
||||
|
||||
### Deploy the model to Azure Kubernetes Service
|
||||
|
||||
|
|
|
|||
|
|
@ -1,113 +0,0 @@
|
|||
# CI/CD pipelines for model train/register and deployment
|
||||
|
||||
Follow this guide to set up two separate pipelines to train/register models and deploy models. This set of pipelines is functionally similar to the [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) pipeline in the [getting started](getting_started.md) guide.
|
||||
|
||||
1. **Model CI, training, evaluation, and registration** - triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage, and publishes and runs the training pipeline. If a new model is registered after evaluation, it creates a build artifact containing the JSON metadata of the model. Definition: [diabetes_regression-train-register.yml](../.pipelines/diabetes_regression-train-register.yml).
|
||||
1. **Release deployment** - consumes the artifact of the previous pipeline and deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. Definition: [diabetes_regression-deploy.yml](../.pipelines/diabetes_regression-deploy.yml).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
---
|
||||
|
||||
It is recommended to go through the [getting started guide](getting_started.md) before starting this guide.
|
||||
|
||||
---
|
||||
|
||||
Before continuing this guide, you will need:
|
||||
|
||||
- An existing workspace. To setup your environment with a new workspace, follow the steps here.
|
||||
- An Azure DevOps Service Connection with your Azure ML Workspace.
|
||||
- A variable group named **``devopsforai-aml-vg``** with the required variables set.
|
||||
|
||||
## Model CI, training, evaluation, and registration pipeline
|
||||
|
||||
In this section, we will create the pipeline that will perform model IC, training, evaluation, and registration.
|
||||
|
||||
### Set up the Pipeline
|
||||
|
||||
In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-ci-train-register.yml](../.pipelines/diabetes_regression-ci-train-register.yml)
|
||||
pipeline definition in your forked repository.
|
||||
|
||||
If you plan to use the release deployment pipeline (in the next section), you will need to rename this pipeline to `Model-Train-Register-CI`.
|
||||
|
||||
Once the pipeline is finished, check the execution result:
|
||||
|
||||
|
||||
|
||||
And the pipeline artifacts:
|
||||
|
||||
|
||||
|
||||
The pipeline stages are summarized below:
|
||||
|
||||
#### Model CI
|
||||
|
||||
- Linting (code quality analysis)
|
||||
- Unit tests and code coverage analysis
|
||||
- Build and publish *ML Training Pipeline* in an *ML Workspace*
|
||||
|
||||
#### Train model
|
||||
|
||||
- Determine the ID of the *ML Training Pipeline* published in the previous stage.
|
||||
- Trigger the *ML Training Pipeline* and waits for it to complete.
|
||||
- This is an **agentless** job. The CI pipeline can wait for ML pipeline completion for hours or even days without using agent resources.
|
||||
- Determine if a new model was registered by the *ML Training Pipeline*.
|
||||
- If the model evaluation determines that the new model doesn't perform any better than the previous one, the new model won't register and the *ML Training Pipeline* will be **canceled**. In this case, you'll see a message in the 'Train Model' job under the 'Determine if evaluation succeeded and new model is registered' step saying '**Model was not registered for this run.**'
|
||||
- See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic.
|
||||
- [Additional Variables and Configuration](getting_started.md#additional-variables-and-configuration) for configuring this and other behavior.
|
||||
|
||||
#### Create pipeline artifact
|
||||
|
||||
- Get the info about the registered model
|
||||
- Create a pipeline artifact called `model` that contains a `model.json` file containing the model information.
|
||||
|
||||
## Release deployment pipeline
|
||||
|
||||
---
|
||||
**PREREQUISITE**
|
||||
|
||||
In order to use this pipeline:
|
||||
|
||||
1. Follow the steps to set up the Model CI, training, evaluation, and registration pipeline.
|
||||
1. You **must** rename your model CI/train/eval/register pipeline to `Model-Train-Register-CI`.
|
||||
|
||||
The release deploymment pipeline relies on the model CI pipeline and references it by name.
|
||||
|
||||
---
|
||||
|
||||
In this section, we will set up the pipeline for release deployment to ACI, AKS, or Webapp. This pipeline uses the resulting artifact of the [Model-Train-Register-CI pipeline](#) to identify the model to be deployed.
|
||||
|
||||
This pipeline has the following behaviors:
|
||||
|
||||
- The pipeline will **automatically trigger** on completion of the Model-Train-Register-CI pipeline
|
||||
- The pipeline will default to using the latest successful build of the Model-Train-Register-CI pipeline. It will deploy the model produced by that build.
|
||||
- You can specify a `Model-Train-Register-CI` build ID when running the pipeline manually. You can find this in the url of the build, and the model registered from that build will also be tagged with the build ID.
|
||||
|
||||
### Set up the pipeline
|
||||
|
||||
In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-cd-deploy.yml](../.pipelines/diabetes_regression-cd-deploy.yml)
|
||||
pipeline definition in your forked repository.
|
||||
|
||||
Your first run will use the latest model created by the `Model-Train-Register-CI` pipeline.
|
||||
|
||||
Once the pipeline is finished, check the execution result:
|
||||
|
||||
|
||||
|
||||
To specify a particular build's model, set the `Model Train CI Build Id` parameter to the build Id you would like to use.
|
||||
|
||||
|
||||
|
||||
Once your pipeline run begins, you can see the model name and version downloaded from the `Model-Train-Register-CI` pipeline.
|
||||
|
||||
|
||||
|
||||
The pipeline has the following stage:
|
||||
|
||||
#### Deploy to ACI
|
||||
|
||||
- Deploy the model to the QA environment in [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/).
|
||||
- Smoke test
|
||||
- The test sends a sample query to the scoring web service and verifies that it returns the expected response. Have a look at the [smoke test code](../ml_service/util/smoke_test_scoring_service.py) for an example.
|
||||
|
||||
See [Further Exploration](getting_started.md#further-exploration) to learn about other deployment targets.
|
||||
Загрузка…
Ссылка в новой задаче