Merge pull request #119 from sdonohoo/main-dec31
Fixes for GitHub Actions Deployments
This commit is contained in:
Scott Donohoo
GitHub
Коммит
b12c8bae2b
|
|
@ -2,64 +2,62 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
variables:
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: aml-cli-v2
|
||||
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: aml-cli-v2
|
||||
|
||||
trigger:
|
||||
- none
|
||||
- none
|
||||
|
||||
pool:
|
||||
vmImage: ubuntu-20.04
|
||||
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: mlops-templates # Template Repo
|
||||
- repository: mlops-templates # Template Repo
|
||||
name: Azure/mlops-templates # need to change org name from "Azure" to your own org
|
||||
endpoint: github-connection # need to set up and hardcode
|
||||
type: github
|
||||
ref: main
|
||||
|
||||
stages:
|
||||
- stage: DeployTrainingPipeline
|
||||
displayName: Deploy Training Pipeline
|
||||
jobs:
|
||||
- job: DeployTrainingPipeline
|
||||
timeoutInMinutes: 120 # how long to run the job before automatically cancelling
|
||||
steps:
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
|
||||
parameters:
|
||||
environment_name: taxi-train-env
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
|
||||
parameters:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: LowPriority
|
||||
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates
|
||||
parameters:
|
||||
data_type: uri_file
|
||||
data_name: taxi-data
|
||||
data_file: mlops/azureml/train/data.yml
|
||||
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
|
||||
parameters:
|
||||
pipeline_file: mlops/azureml/train/pipeline.yml
|
||||
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
|
||||
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
|
||||
enable_monitoring: $(enable_monitoring)
|
||||
- stage: DeployTrainingPipeline
|
||||
displayName: Deploy Training Pipeline
|
||||
jobs:
|
||||
- job: DeployTrainingPipeline
|
||||
timeoutInMinutes: 120 # how long to run the job before automatically cancelling
|
||||
steps:
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
|
||||
parameters:
|
||||
environment_name: taxi-train-env
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
|
||||
parameters:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: low_priority
|
||||
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates
|
||||
parameters:
|
||||
data_type: uri_file
|
||||
data_name: taxi-data
|
||||
data_file: mlops/azureml/train/data.yml
|
||||
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
|
||||
parameters:
|
||||
pipeline_file: mlops/azureml/train/pipeline.yml
|
||||
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
|
||||
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
|
||||
enable_monitoring: $(enable_monitoring)
|
||||
|
|
|
|||
|
|
@ -1,44 +1,63 @@
|
|||
name: deploy-batch-endpoint-pipeline
|
||||
|
||||
on:
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
|
||||
with:
|
||||
file_name: config-infra-prod.yml
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
create-compute:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main
|
||||
with:
|
||||
cluster_name: batch-cluster
|
||||
size: STANDARD_DS3_V2
|
||||
min_instances: 0
|
||||
max_instances: 5
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
cluster_name: batch-cluster
|
||||
size: STANDARD_DS3_V2
|
||||
min_instances: 0
|
||||
max_instances: 5
|
||||
cluster_tier: low_priority
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
create-endpoint:
|
||||
needs: [get-config,create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/create-endpoint.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/batch/batch-endpoint.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.bep) }}
|
||||
endpoint_type: batch
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: [get-config, create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/create-endpoint.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/batch/batch-endpoint.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.bep) }}
|
||||
endpoint_type: batch
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
create-deployment:
|
||||
uses: Azure/mlops-templates/.github/workflows/create-deployment.yml@main
|
||||
needs: [get-config,create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/batch/batch-deployment.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.bep) }}
|
||||
endpoint_type: batch
|
||||
deployment_name: eptestdeploy
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
uses: Azure/mlops-templates/.github/workflows/create-deployment.yml@main-dec31 # TODO - revert to @main
|
||||
needs: [get-config, create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/batch/batch-deployment.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.bep) }}
|
||||
endpoint_type: batch
|
||||
deployment_name: eptestdeploy
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
|
|
|
|||
|
|
@ -1,42 +1,70 @@
|
|||
name: deploy-model-training-pipeline
|
||||
|
||||
on:
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
|
||||
with:
|
||||
file_name: config-infra-prod.yml
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
register-environment:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/register-environment.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
conda_file: data-science/environment/train-conda.yml
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/register-environment.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
conda_file: data-science/environment/train-conda.yml
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
register-dataset:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/register-dataset.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
name: taxi-data
|
||||
data_file: mlops/azureml/train/data.yml
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
create-compute:
|
||||
needs: [get-config,register-environment]
|
||||
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main
|
||||
with:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: LowPriority
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
run-pipeline:
|
||||
needs: [get-config,register-environment,create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/run-pipeline.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
parameters-file: mlops/azureml/train/pipeline.yml
|
||||
job-name: test
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: [get-config]
|
||||
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: low_priority
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
run-model-training-pipeline:
|
||||
needs: [get-config, register-environment, register-dataset, create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/run-pipeline.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
parameters-file: mlops/azureml/train/pipeline.yml
|
||||
job-name: test
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
|
|
|
|||
|
|
@ -1,42 +1,60 @@
|
|||
name: deploy-online-endpoint-pipeline
|
||||
|
||||
on:
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
|
||||
with:
|
||||
file_name: config-infra-prod.yml
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
create-endpoint:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/create-endpoint.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-endpoint.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
endpoint_type: online
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/create-endpoint.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-endpoint.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
endpoint_type: online
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
create-deployment:
|
||||
uses: Azure/mlops-templates/.github/workflows/create-deployment.yml@main
|
||||
needs: [get-config,create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-deployment.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
endpoint_type: online
|
||||
deployment_name: taxi-online-dp
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
uses: Azure/mlops-templates/.github/workflows/create-deployment.yml@main-dec31 # TODO - revert to @main
|
||||
needs: [get-config, create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-deployment.yml
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
endpoint_type: online
|
||||
deployment_name: taxi-online-dp
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
allocate-traffic:
|
||||
uses: Azure/mlops-templates/.github/workflows/allocate-traffic.yml@main
|
||||
needs: [get-config,create-deployment]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
traffic_allocation: taxi-online-dp=100
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
uses: Azure/mlops-templates/.github/workflows/allocate-traffic.yml@main-dec31 # TODO - revert to @main
|
||||
needs: [get-config, create-deployment]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
traffic_allocation: taxi-online-dp=100
|
||||
endpoint_name: ${{ format('taxi-gha-{0}', needs.get-config.outputs.oep) }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
|
|
|
|||
|
|
@ -2,25 +2,24 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
variables:
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: python-sdk-v2
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: python-sdk-v2
|
||||
|
||||
trigger:
|
||||
- none
|
||||
- none
|
||||
|
||||
pool:
|
||||
vmImage: ubuntu-20.04
|
||||
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: mlops-templates # Template Repo
|
||||
- repository: mlops-templates # Template Repo
|
||||
name: Azure/mlops-templates # need to change org name from "Azure" to your own org
|
||||
endpoint: github-connection # need to set up and hardcode
|
||||
type: github
|
||||
|
|
@ -47,7 +46,7 @@ stages:
|
|||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: LowPriority
|
||||
cluster_tier: low_priority
|
||||
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
|
||||
parameters:
|
||||
environment_name: taxi-train-env
|
||||
|
|
|
|||
|
|
@ -2,77 +2,75 @@
|
|||
# Licensed under the MIT License.
|
||||
|
||||
variables:
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: aml-cli-v2
|
||||
|
||||
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'main' branch: PRD environment
|
||||
- template: ../../config-infra-prod.yml
|
||||
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
|
||||
# 'develop' or feature branches: DEV environment
|
||||
- template: ../../config-infra-dev.yml
|
||||
- name: version
|
||||
value: aml-cli-v2
|
||||
|
||||
trigger:
|
||||
- none
|
||||
- none
|
||||
|
||||
pool:
|
||||
vmImage: ubuntu-20.04
|
||||
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: mlops-templates # Template Repo
|
||||
- repository: mlops-templates # Template Repo
|
||||
name: Azure/mlops-templates # need to change org name from "Azure" to your own org
|
||||
endpoint: github-connection # need to set up and hardcode
|
||||
type: github
|
||||
ref: main
|
||||
- repository: rai-vnext-preview # Template Repo
|
||||
- repository: rai-vnext-preview # Template Repo
|
||||
name: Azure/rai-vnext-preview # need to change org name from "Azure" to your own org
|
||||
endpoint: github-connection # need to set up and hardcode
|
||||
type: github
|
||||
ref: main
|
||||
|
||||
stages:
|
||||
- stage: DeployTrainingPipeline
|
||||
displayName: Deploy Training Pipeline
|
||||
jobs:
|
||||
- job: DeployTrainingPipeline
|
||||
timeoutInMinutes: 120 # how long to run the job before automatically cancelling
|
||||
steps:
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/tests/unit-tests.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
|
||||
parameters:
|
||||
environment_name: taxi-train-env
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
|
||||
parameters:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: LowPriority
|
||||
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates
|
||||
parameters:
|
||||
data_type: uri_file
|
||||
data_name: taxi-data
|
||||
data_file: mlops/azureml/train/data.yml
|
||||
- checkout: rai-vnext-preview
|
||||
path: s/
|
||||
- template: register-rai-components.yml
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
|
||||
parameters:
|
||||
pipeline_file: mlops/azureml/train/pipeline.yml
|
||||
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
|
||||
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
|
||||
enable_monitoring: $(enable_monitoring)
|
||||
- stage: DeployTrainingPipeline
|
||||
displayName: Deploy Training Pipeline
|
||||
jobs:
|
||||
- job: DeployTrainingPipeline
|
||||
timeoutInMinutes: 120 # how long to run the job before automatically cancelling
|
||||
steps:
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/tests/unit-tests.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-az-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/install-aml-cli.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
|
||||
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
|
||||
parameters:
|
||||
environment_name: taxi-train-env
|
||||
environment_file: mlops/azureml/train/train-env.yml
|
||||
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
|
||||
parameters:
|
||||
cluster_name: cpu-cluster
|
||||
size: Standard_DS3_v2
|
||||
min_instances: 0
|
||||
max_instances: 4
|
||||
cluster_tier: low_priority
|
||||
- template: templates/${{ variables.version }}/register-data.yml@mlops-templates
|
||||
parameters:
|
||||
data_type: uri_file
|
||||
data_name: taxi-data
|
||||
data_file: mlops/azureml/train/data.yml
|
||||
- checkout: rai-vnext-preview
|
||||
path: s/
|
||||
- template: register-rai-components.yml
|
||||
- checkout: self
|
||||
path: s/
|
||||
- checkout: mlops-templates
|
||||
path: s/templates/
|
||||
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
|
||||
parameters:
|
||||
pipeline_file: mlops/azureml/train/pipeline.yml
|
||||
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
|
||||
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
|
||||
enable_monitoring: $(enable_monitoring)
|
||||
|
|
|
|||
|
|
@ -1,18 +1,16 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
# Dev environment
|
||||
variables:
|
||||
|
||||
# Global
|
||||
ap_vm_image: ubuntu-20.04
|
||||
|
||||
namespace: mlopsv2 #Note: A namespace with many characters will cause storage account creation to fail due to storage account names having a limit of 24 characters.
|
||||
postfix: mb04
|
||||
location: westus
|
||||
|
||||
postfix: 0001
|
||||
location: eastus
|
||||
environment: dev
|
||||
enable_aml_computecluster: true
|
||||
enable_aml_secure_workspace: true
|
||||
enable_monitoring: false
|
||||
|
||||
# Azure DevOps
|
||||
|
|
@ -30,7 +28,6 @@ variables:
|
|||
storage_account: st$(namespace)$(postfix)$(environment)
|
||||
|
||||
# For terraform reference
|
||||
# terraform_version: 0.14.7 - quite old - updating below
|
||||
terraform_version: 1.3.6
|
||||
terraform_workingdir: infrastructure
|
||||
terraform_st_location: $(location)
|
||||
|
|
|
|||
|
|
@ -3,18 +3,16 @@
|
|||
|
||||
# Prod environment
|
||||
variables:
|
||||
|
||||
# Global
|
||||
ap_vm_image: ubuntu-20.04
|
||||
|
||||
namespace: mlopsv2 #Note: A namespace with many characters will cause storage account creation to fail due to storage account names having a limit of 24 characters.
|
||||
postfix: 001
|
||||
location: uksouth
|
||||
postfix: 0001
|
||||
location: eastus
|
||||
environment: prod
|
||||
enable_aml_computecluster: true
|
||||
enable_monitoring: false
|
||||
|
||||
|
||||
# Azure DevOps
|
||||
ado_service_connection_rg: Azure-ARM-Prod
|
||||
ado_service_connection_aml_ws: Azure-ARM-Prod
|
||||
|
|
@ -32,6 +30,7 @@ variables:
|
|||
# For terraform reference
|
||||
terraform_version: 1.3.6
|
||||
terraform_workingdir: infrastructure
|
||||
terraform_st_location: $(location)
|
||||
terraform_st_resource_group: rg-$(namespace)-$(postfix)$(environment)-tf
|
||||
terraform_st_storage_account: st$(namespace)$(postfix)$(environment)tf
|
||||
terraform_st_container_name: default
|
||||
|
|
|
|||
|
|
@ -1,51 +1,69 @@
|
|||
name: deploy-cv-model-training-pipeline
|
||||
|
||||
on:
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
|
||||
with:
|
||||
file_name: config-infra-prod.yml
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
create-compute:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-compute.yml@main
|
||||
with:
|
||||
cluster_name: gpu-cluster
|
||||
size: Standard_NC6
|
||||
min_instances: 0
|
||||
max_instances: 1
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-compute.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
cluster_name: gpu-cluster
|
||||
size: Standard_NC6
|
||||
min_instances: 0
|
||||
max_instances: 1
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
register-environment:
|
||||
needs: [get-config,create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-register-environment.yml
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
environment_file: mlops/azureml/train/train-env.yaml
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: [get-config, create-compute]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-register-environment.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
environment_file: mlops/azureml/train/train-env.yaml
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
register-dataset:
|
||||
needs: [get-config,register-environment]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-register-dataset.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
data_file: mlops/azureml/train/create_stanford_dogs_dataset.yaml
|
||||
file_type: Training
|
||||
name: stanford_dogs
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: [get-config, register-environment]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-register-dataset.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
data_file: mlops/azureml/train/create_stanford_dogs_dataset.yaml
|
||||
file_type: Training
|
||||
name: stanford_dogs
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
run-pipeline:
|
||||
needs: [get-config,register-dataset]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-run-pipeline.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
parameters-file: mlops/azureml/train/pipeline.yaml
|
||||
job-name: cv-train
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: [get-config, register-dataset]
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-run-pipeline.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
parameters-file: mlops/azureml/train/pipeline.yaml
|
||||
job-name: cv-train
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
|
|
|
|||
|
|
@ -1,42 +1,60 @@
|
|||
name: deploy-online-endpoint-pipeline
|
||||
|
||||
on:
|
||||
on:
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main
|
||||
with:
|
||||
file_name: config-infra-prod.yml
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
create-endpoint:
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-endpoint.yml@main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-endpoint.yml
|
||||
endpoint_name: dogs-classifier-online2
|
||||
endpoint_type: online
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
needs: get-config
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-endpoint.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-endpoint.yml
|
||||
endpoint_name: dogs-classifier-online2
|
||||
endpoint_type: online
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
create-deployment:
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-deployment.yml@main
|
||||
needs: [get-config,create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-deployment.yml
|
||||
endpoint_name: dogs-classifier-online2
|
||||
endpoint_type: online
|
||||
deployment_name: dogs-online-dp
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-create-deployment.yml@main-dec31 # TODO - revert to @main
|
||||
needs: [get-config, create-endpoint]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
endpoint_file: mlops/azureml/deploy/online/online-deployment.yml
|
||||
endpoint_name: dogs-classifier-online2
|
||||
endpoint_type: online
|
||||
deployment_name: dogs-online-dp
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
allocate-traffic:
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-allocate-traffic.yml@main
|
||||
needs: [get-config,create-deployment]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
traffic_allocation: dogs-online-dp=100
|
||||
endpoint_name: dogs-classifier-online2
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
uses: Azure/mlops-templates/.github/workflows/cli-v2-allocate-traffic.yml@main-dec31 # TODO - revert to @main
|
||||
needs: [get-config, create-deployment]
|
||||
with:
|
||||
resource_group: ${{ needs.get-config.outputs.resource_group }}
|
||||
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
|
||||
traffic_allocation: dogs-online-dp=100
|
||||
endpoint_name: dogs-classifier-online2
|
||||
secrets:
|
||||
creds: ${{secrets.AZURE_CREDENTIALS}}
|
||||
|
|
|
|||
|
|
@ -1,42 +1,39 @@
|
|||
name: tf-gha-deploy-infra.yml
|
||||
|
||||
on:
|
||||
push:
|
||||
|
||||
on:
|
||||
#push:
|
||||
workflow_dispatch:
|
||||
env:
|
||||
config_env: 'none'
|
||||
config_env: "none"
|
||||
jobs:
|
||||
set-env-branch:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
config-file: ${{ steps.set-output-defaults.outputs.config-file }}
|
||||
steps:
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/head/main'}}
|
||||
run:
|
||||
echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/head/main'}}
|
||||
run:
|
||||
echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
- id: set-prod-branch
|
||||
name: set-prod-branch
|
||||
if: ${{ github.ref == 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-prod.yml" >> $GITHUB_ENV;
|
||||
- id: set-dev-branch
|
||||
name: setdevbranch
|
||||
if: ${{ github.ref != 'refs/heads/main'}}
|
||||
run: echo "config_env=config-infra-dev.yml" >> $GITHUB_ENV;
|
||||
- id: set-output-defaults
|
||||
name: set-output-defaults
|
||||
run: |
|
||||
echo "config-file=$config_env" >> $GITHUB_OUTPUT;
|
||||
get-config:
|
||||
needs: set-env-branch
|
||||
uses: Azure/mlops-templates/.github/workflows/read-yaml.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
with:
|
||||
file_name: ${{ needs.set-env-branch.outputs.config-file}}
|
||||
test-terraform-state-deployment:
|
||||
needs: [get-config,set-env-branch]
|
||||
needs: [get-config, set-env-branch]
|
||||
uses: Azure/mlops-templates/.github/workflows/tf-gha-install-terraform.yml@main-dec31 # TODO - revert to @main
|
||||
with:
|
||||
with:
|
||||
TFAction: 'apply'
|
||||
dply_environment: ${{ needs.set-env-branch.outputs.config-file }}
|
||||
TFAction: "apply"
|
||||
dply_environment: ${{ needs.set-env-branch.outputs.config-file }}
|
||||
location: ${{ needs.get-config.outputs.location }}
|
||||
namespace: ${{ needs.get-config.outputs.namespace }}
|
||||
postfix: ${{ needs.get-config.outputs.postfix }}
|
||||
|
|
@ -61,7 +58,6 @@ jobs:
|
|||
deploy-azureml-resources:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- id: deploy-aml-workspace
|
||||
name: deploy-aml-workspace
|
||||
run: echo "OK"
|
||||
|
||||
- id: deploy-aml-workspace
|
||||
name: deploy-aml-workspace
|
||||
run: echo "OK"
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче