110 строки
4.1 KiB
YAML
110 строки
4.1 KiB
YAML
# Copyright (c) Microsoft Corporation.
|
|
# Licensed under the MIT license.
|
|
|
|
|
|
name: $(Date:yyyyMMdd)$(Rev:.r)
|
|
parameters:
|
|
- name: KEYVAULT_NAME
|
|
displayName: Keyvault Name
|
|
type: string
|
|
default: cipipelineresources
|
|
- name: AZURE_REGION
|
|
displayName: Azure Region
|
|
type: string
|
|
default: eastus
|
|
|
|
variables:
|
|
- name: serviceConnectionToAzure
|
|
value: 'github-az-creds'
|
|
- name: DisableDockerDetector # To disable public docker image checking
|
|
value: true
|
|
|
|
pr:
|
|
branches:
|
|
include:
|
|
- main
|
|
- releases/*
|
|
paths:
|
|
exclude:
|
|
- '*.md'
|
|
- docs
|
|
- deploy/README.md
|
|
|
|
trigger:
|
|
batch: true
|
|
branches:
|
|
include:
|
|
- '*'
|
|
paths:
|
|
exclude:
|
|
- '*.md'
|
|
- docs
|
|
- deploy/README.md
|
|
|
|
pool:
|
|
vmImage: ubuntu-latest
|
|
|
|
jobs:
|
|
- job: RunE2ETest
|
|
displayName: "Deploy Infrastructure and run Synapse Pipeline"
|
|
timeoutInMinutes: 300
|
|
steps:
|
|
- task: AzureCLI@1
|
|
name: DeployAndTest
|
|
inputs:
|
|
azureSubscription: '$(serviceConnectionToAzure)'
|
|
scriptLocation: inlineScript
|
|
inlineScript: |
|
|
set -e
|
|
|
|
export KEYVAULT_NAME=${{ parameters.KEYVAULT_NAME }}
|
|
export AZURE_REGION=${{ parameters.AZURE_REGION }}
|
|
|
|
export AAD_GROUP_PRINCIPAL_ID="$(az keyvault secret show --vault ${KEYVAULT_NAME} --name AAD-GROUP-PRINCIPAL-ID --query value -o tsv)"
|
|
export CI_BATCH_ACCOUNT_NAME="$(az keyvault secret show --vault ${KEYVAULT_NAME} --name CI-BATCH-ACCOUNT-NAME --query value -o tsv)"
|
|
export CI_STORAGE_ACCOUNT_NAME="$(az keyvault secret show --vault ${KEYVAULT_NAME} --name CI-STORAGE-ACCOUNT --query value -o tsv)"
|
|
|
|
MD5_OUTPUT="$(echo $(date +%s) | md5sum | awk '{print $1}')"
|
|
ENV_CODE=$(echo pr${MD5_OUTPUT:0:7})
|
|
|
|
echo "##vso[task.setvariable variable=environment_code;isOutput=true]${ENV_CODE}"
|
|
echo "##vso[task.setvariable variable=batch_account_name;isOutput=true]${CI_BATCH_ACCOUNT_NAME}"
|
|
|
|
chars='@#$%&_+='
|
|
export POSTGRES_ADMIN_LOGIN_PASS=$({ </dev/urandom LC_ALL=C grep -ao '[A-Za-z0-9]' | head -n$((RANDOM % 8 + 9)); echo ${chars:$((RANDOM % ${#chars})):1}; } | shuf | tr -d '\n')
|
|
|
|
# Setup the infrastructure for testing the pipeline
|
|
DEPLOY_PGSQL=${USE_POSTGRE_SQL} \
|
|
PRE_PROVISIONED_AI_MODEL_INFRA_NAME=${CI_BATCH_ACCOUNT_NAME} \
|
|
$(System.DefaultWorkingDirectory)/deploy/scripts/setup.sh ${ENV_CODE} ${AZURE_REGION} ${ENV_CODE}-ci-environment
|
|
|
|
# Grant access on Synapse workspace to resource owner azure account
|
|
SYNAPSE_WORKSPACE_NAME=$(az synapse workspace list --query "[?tags.workspaceId && tags.workspaceId == 'default'].name" -o tsv -g "${ENV_CODE}-pipeline-rg")
|
|
echo "Granting access to resource owner azure account on synapse workspace ${SYNAPSE_WORKSPACE_NAME}"
|
|
az synapse role assignment create --workspace-name ${SYNAPSE_WORKSPACE_NAME} --role "Synapse Administrator" --assignee $(az account show --query 'user.name' -o tsv)
|
|
# Granting access on Synapse workspace for AAD group
|
|
echo "Granting access to the AAD group on synapse workspace ${SYNAPSE_WORKSPACE_NAME}"
|
|
az synapse role assignment create --workspace-name ${SYNAPSE_WORKSPACE_NAME} --role "Synapse Administrator" --assignee ${AAD_GROUP_PRINCIPAL_ID}
|
|
|
|
# Setup and Run Tests
|
|
BATCH_ACCOUNT_NAME=${CI_BATCH_ACCOUNT_NAME} SOURCE_STORAGE_ACCOUNT_NAME=${CI_STORAGE_ACCOUNT_NAME} $(System.DefaultWorkingDirectory)/test/deploy-prepare-and-test-pipeline.sh ${ENV_CODE}
|
|
addSpnToEnvironment: true
|
|
useGlobalConfig: true
|
|
- job: CleanupInfrastructure
|
|
displayName: "Cleanup Azure Infrastructure"
|
|
dependsOn: RunE2ETest
|
|
condition: always()
|
|
variables:
|
|
ENV_CODE: $[ dependencies.RunE2ETest.outputs['DeployAndTest.environment_code'] ]
|
|
CI_BATCH_ACCOUNT_NAME: $[ dependencies.RunE2ETest.outputs['DeployAndTest.batch_account_name'] ]
|
|
timeoutInMinutes: 300
|
|
steps:
|
|
- task: AzureCLI@1
|
|
inputs:
|
|
azureSubscription: '$(serviceConnectionToAzure)'
|
|
scriptLocation: inlineScript
|
|
inlineScript: |
|
|
set -e
|
|
# Cleanup the resources
|
|
$(System.DefaultWorkingDirectory)/deploy/scripts/cleanup.sh ${ENV_CODE} ${CI_BATCH_ACCOUNT_NAME}
|