Azure
/
mlops-project-template
зеркало из https://github.com/Azure/mlops-project-template.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Fix incorrect low_priority vm spec

This commit is contained in:
Scott Donohoo 2022-12-20 11:39:25 -06:00
Родитель c3f1e42e1e
Коммит 22a2a9ba41
4 изменённых файлов: 183 добавлений и 188 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

92
classical/aml-cli-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml
Просмотреть файл

@ -2,64 +2,62 @@
# Licensed under the MIT License. # Licensed under the MIT License.
variables: variables:
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}: - ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
# 'main' branch: PRD environment # 'main' branch: PRD environment
- template: ../../config-infra-prod.yml - template: ../../config-infra-prod.yml
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}: - ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
# 'develop' or feature branches: DEV environment # 'develop' or feature branches: DEV environment
- template: ../../config-infra-dev.yml - template: ../../config-infra-dev.yml
- name: version - name: version
value: aml-cli-v2 value: aml-cli-v2
trigger: trigger:
- none - none
pool: pool:
vmImage: ubuntu-20.04 vmImage: ubuntu-20.04
resources: resources:
repositories: repositories:
- repository: mlops-templates # Template Repo - repository: mlops-templates # Template Repo
name: Azure/mlops-templates # need to change org name from "Azure" to your own org name: Azure/mlops-templates # need to change org name from "Azure" to your own org
endpoint: github-connection # need to set up and hardcode endpoint: github-connection # need to set up and hardcode
type: github type: github
ref: main ref: main
stages: stages:
- stage: DeployTrainingPipeline - stage: DeployTrainingPipeline
displayName: Deploy Training Pipeline displayName: Deploy Training Pipeline
jobs: jobs:
- job: DeployTrainingPipeline - job: DeployTrainingPipeline
timeoutInMinutes: 120 # how long to run the job before automatically cancelling timeoutInMinutes: 120 # how long to run the job before automatically cancelling
steps: steps:
- checkout: self - checkout: self
path: s/ path: s/
- checkout: mlops-templates - checkout: mlops-templates
path: s/templates/ path: s/templates/
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates - template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates - template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates - template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates - template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
parameters: parameters:
environment_name: taxi-train-env environment_name: taxi-train-env
environment_file: mlops/azureml/train/train-env.yml environment_file: mlops/azureml/train/train-env.yml
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates - template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
parameters: parameters:
cluster_name: cpu-cluster cluster_name: cpu-cluster
size: Standard_DS3_v2 size: Standard_DS3_v2
min_instances: 0 min_instances: 0
max_instances: 4 max_instances: 4
cluster_tier: LowPriority cluster_tier: low_priority
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates - template: templates/${{ variables.version }}/register-data.yml@mlops-templates
parameters: parameters:
data_type: uri_file data_type: uri_file
data_name: taxi-data data_name: taxi-data
data_file: mlops/azureml/train/data.yml data_file: mlops/azureml/train/data.yml
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates - template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
parameters: parameters:
pipeline_file: mlops/azureml/train/pipeline.yml pipeline_file: mlops/azureml/train/pipeline.yml
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName) experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
display_name: $(environment)_taxi_fare_run_$(Build.BuildID) display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
enable_monitoring: $(enable_monitoring) enable_monitoring: $(enable_monitoring)

68
classical/aml-cli-v2/mlops/github-actions/deploy-model-training-pipeline-classical.yml
Просмотреть файл

@ -1,42 +1,42 @@
name: deploy-model-training-pipeline name: deploy-model-training-pipeline
on: on:
workflow_dispatch: workflow_dispatch:
jobs: jobs:
get-config: get-config:
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
with: with:
file_name: config-infra-prod.yml file_name: config-infra-prod.yml
register-environment: register-environment:
needs: get-config needs: get-config
uses: Azure/mlops-templates/.github/workflows/register-environment.yml@main uses: Azure/mlops-templates/.github/workflows/register-environment.yml@main
with: with:
resource_group: ${{ needs.get-config.outputs.resource_group }} resource_group: ${{ needs.get-config.outputs.resource_group }}
workspace_name: ${{ needs.get-config.outputs.aml_workspace }} workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
environment_file: mlops/azureml/train/train-env.yml environment_file: mlops/azureml/train/train-env.yml
conda_file: data-science/environment/train-conda.yml conda_file: data-science/environment/train-conda.yml
secrets: secrets:
creds: ${{secrets.AZURE_CREDENTIALS}} creds: ${{secrets.AZURE_CREDENTIALS}}
create-compute: create-compute:
needs: [get-config,register-environment] needs: [get-config, register-environment]
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main
with: with:
cluster_name: cpu-cluster cluster_name: cpu-cluster
size: Standard_DS3_v2 size: Standard_DS3_v2
min_instances: 0 min_instances: 0
max_instances: 4 max_instances: 4
cluster_tier: LowPriority cluster_tier: low_priority
resource_group: ${{ needs.get-config.outputs.resource_group }} resource_group: ${{ needs.get-config.outputs.resource_group }}
workspace_name: ${{ needs.get-config.outputs.aml_workspace }} workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
secrets: secrets:
creds: ${{secrets.AZURE_CREDENTIALS}} creds: ${{secrets.AZURE_CREDENTIALS}}
run-pipeline: run-pipeline:
needs: [get-config,register-environment,create-compute] needs: [get-config, register-environment, create-compute]
uses: Azure/mlops-templates/.github/workflows/run-pipeline.yml@main uses: Azure/mlops-templates/.github/workflows/run-pipeline.yml@main
with: with:
resource_group: ${{ needs.get-config.outputs.resource_group }} resource_group: ${{ needs.get-config.outputs.resource_group }}
workspace_name: ${{ needs.get-config.outputs.aml_workspace }} workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
parameters-file: mlops/azureml/train/pipeline.yml parameters-file: mlops/azureml/train/pipeline.yml
job-name: test job-name: test
secrets: secrets:
creds: ${{secrets.AZURE_CREDENTIALS}} creds: ${{secrets.AZURE_CREDENTIALS}}

101
classical/python-sdk-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml
Просмотреть файл

@ -2,68 +2,67 @@
# Licensed under the MIT License. # Licensed under the MIT License.
variables: variables:
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}: - ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
# 'main' branch: PRD environment # 'main' branch: PRD environment
- template: ../../config-infra-prod.yml - template: ../../config-infra-prod.yml
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}: - ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
# 'develop' or feature branches: DEV environment # 'develop' or feature branches: DEV environment
- template: ../../config-infra-dev.yml - template: ../../config-infra-dev.yml
- name: version - name: version
value: python-sdk-v2 value: python-sdk-v2
trigger: trigger:
- none - none
pool: pool:
vmImage: ubuntu-20.04 vmImage: ubuntu-20.04
resources: resources:
repositories: repositories:
- repository: mlops-templates # Template Repo - repository: mlops-templates # Template Repo
name: Azure/mlops-templates # need to change org name from "Azure" to your own org name: Azure/mlops-templates # need to change org name from "Azure" to your own org
endpoint: github-connection # need to set up and hardcode endpoint: github-connection # need to set up and hardcode
type: github type: github
ref: main-dec31 ref: main-dec31
stages: stages:
- stage: DeployTrainingPipeline - stage: DeployTrainingPipeline
displayName: Deploy Training Pipeline displayName: Deploy Training Pipeline
jobs: jobs:
- job: DeployTrainingPipeline - job: DeployTrainingPipeline
timeoutInMinutes: 120 # how long to run the job before automatically cancelling timeoutInMinutes: 120 # how long to run the job before automatically cancelling
steps: steps:
- checkout: self - checkout: self
path: s/ path: s/
- checkout: mlops-templates - checkout: mlops-templates
path: s/templates/ path: s/templates/
- template: templates/aml-cli-v2/install-az-cli.yml@mlops-templates - template: templates/aml-cli-v2/install-az-cli.yml@mlops-templates
- template: templates/aml-cli-v2/install-aml-cli.yml@mlops-templates - template: templates/aml-cli-v2/install-aml-cli.yml@mlops-templates
- template: templates/python-sdk-v2/install-requirements.yml@mlops-templates - template: templates/python-sdk-v2/install-requirements.yml@mlops-templates
- template: templates/aml-cli-v2/connect-to-workspace.yml@mlops-templates - template: templates/aml-cli-v2/connect-to-workspace.yml@mlops-templates
- template: templates/aml-cli-v2/create-compute.yml@mlops-templates - template: templates/aml-cli-v2/create-compute.yml@mlops-templates
parameters: parameters:
cluster_name: cpu-cluster cluster_name: cpu-cluster
size: Standard_DS3_v2 size: Standard_DS3_v2
min_instances: 0 min_instances: 0
max_instances: 4 max_instances: 4
cluster_tier: LowPriority cluster_tier: low_priority
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates - template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
parameters: parameters:
environment_name: taxi-train-env environment_name: taxi-train-env
environment_description: "Training Environment for Taxi Pipeline" environment_description: "Training Environment for Taxi Pipeline"
environment_path: data-science/environment/train-conda.yml environment_path: data-science/environment/train-conda.yml
build_type: conda build_type: conda
- template: templates/${{ variables.version }}/register-data-asset.yml@mlops-templates - template: templates/${{ variables.version }}/register-data-asset.yml@mlops-templates
parameters: parameters:
data_name: taxi-data data_name: taxi-data
data_description: taxi-training-dataset data_description: taxi-training-dataset
data_path: data/taxi-data.csv data_path: data/taxi-data.csv
data_type: uri_file data_type: uri_file
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates - template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
parameters: parameters:
pipeline_path: mlops/azureml/train/run_pipeline.py pipeline_path: mlops/azureml/train/run_pipeline.py
data_name: taxi-data data_name: taxi-data
cluster_name: cpu-cluster cluster_name: cpu-cluster
enable_monitoring: $(enable_monitoring) enable_monitoring: $(enable_monitoring)
experiment_name: taxi-train-pipeline experiment_name: taxi-train-pipeline

110
classical/rai-aml-cli-v2/mlops/devops-pipelines/deploy-model-training-pipeline.yml
Просмотреть файл

@ -2,77 +2,75 @@
# Licensed under the MIT License. # Licensed under the MIT License.
variables: variables:
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}: - ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
# 'main' branch: PRD environment # 'main' branch: PRD environment
- template: ../../config-infra-prod.yml - template: ../../config-infra-prod.yml
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}: - ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
# 'develop' or feature branches: DEV environment # 'develop' or feature branches: DEV environment
- template: ../../config-infra-dev.yml - template: ../../config-infra-dev.yml
- name: version - name: version
value: aml-cli-v2 value: aml-cli-v2
trigger: trigger:
- none - none
pool: pool:
vmImage: ubuntu-20.04 vmImage: ubuntu-20.04
resources: resources:
repositories: repositories:
- repository: mlops-templates # Template Repo - repository: mlops-templates # Template Repo
name: Azure/mlops-templates # need to change org name from "Azure" to your own org name: Azure/mlops-templates # need to change org name from "Azure" to your own org
endpoint: github-connection # need to set up and hardcode endpoint: github-connection # need to set up and hardcode
type: github type: github
ref: main ref: main
- repository: rai-vnext-preview # Template Repo - repository: rai-vnext-preview # Template Repo
name: Azure/rai-vnext-preview # need to change org name from "Azure" to your own org name: Azure/rai-vnext-preview # need to change org name from "Azure" to your own org
endpoint: github-connection # need to set up and hardcode endpoint: github-connection # need to set up and hardcode
type: github type: github
ref: main ref: main
stages: stages:
- stage: DeployTrainingPipeline - stage: DeployTrainingPipeline
displayName: Deploy Training Pipeline displayName: Deploy Training Pipeline
jobs: jobs:
- job: DeployTrainingPipeline - job: DeployTrainingPipeline
timeoutInMinutes: 120 # how long to run the job before automatically cancelling timeoutInMinutes: 120 # how long to run the job before automatically cancelling
steps: steps:
- checkout: self - checkout: self
path: s/ path: s/
- checkout: mlops-templates - checkout: mlops-templates
path: s/templates/ path: s/templates/
- template: templates/tests/unit-tests.yml@mlops-templates - template: templates/tests/unit-tests.yml@mlops-templates
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates - template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates - template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates - template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates - template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
parameters: parameters:
environment_name: taxi-train-env environment_name: taxi-train-env
environment_file: mlops/azureml/train/train-env.yml environment_file: mlops/azureml/train/train-env.yml
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates - template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
parameters: parameters:
cluster_name: cpu-cluster cluster_name: cpu-cluster
size: Standard_DS3_v2 size: Standard_DS3_v2
min_instances: 0 min_instances: 0
max_instances: 4 max_instances: 4
cluster_tier: LowPriority cluster_tier: low_priority
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates - template: templates/${{ variables.version }}/register-data.yml@mlops-templates
parameters: parameters:
data_type: uri_file data_type: uri_file
data_name: taxi-data data_name: taxi-data
data_file: mlops/azureml/train/data.yml data_file: mlops/azureml/train/data.yml
- checkout: rai-vnext-preview - checkout: rai-vnext-preview
path: s/ path: s/
- template: register-rai-components.yml - template: register-rai-components.yml
- checkout: self - checkout: self
path: s/ path: s/
- checkout: mlops-templates - checkout: mlops-templates
path: s/templates/ path: s/templates/
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates - template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
parameters: parameters:
pipeline_file: mlops/azureml/train/pipeline.yml pipeline_file: mlops/azureml/train/pipeline.yml
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName) experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
display_name: $(environment)_taxi_fare_run_$(Build.BuildID) display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
enable_monitoring: $(enable_monitoring) enable_monitoring: $(enable_monitoring)