Azure
/
mlops-infra-default
зеркало из https://github.com/Azure/mlops-infra-default.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merging the complete build with online and batch end points. (#15) 

* Removed dummy files and added actual files for training pipeline.
* Organizing artifactstore
* Set up CI with Azure Pipelines
* Updated the service connection name for the template to run.
* Update deploy-model-training-pipeline-v2.yml for Azure Pipelines


Co-authored-by: cindyweng <weng.cindy@gmail.com>
Co-authored-by: Cindy Weng <8880364+cindyweng@users.noreply.github.com>
Co-authored-by: murggu <amurguzur@gmail.com>
Co-authored-by: Maggie Mhanna <maggiemhanna@gmail.com>
Co-authored-by: Christoph Muller-Reyes <chrey@microsoft.com>
Co-authored-by: chrey-gh <58181624+chrey-gh@users.noreply.github.com>
This commit is contained in:
Setu Chokshi 2022-04-13 18:56:04 +05:30 коммит произвёл GitHub
Родитель a972eff821
Коммит aa58270749
34 изменённых файлов: 1395 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
templates/sample.yml → arm/.gitkeep
Просмотреть файл

0
bicep/.gitkeep Normal file
Просмотреть файл

93
bicep/main.bicep Normal file
Просмотреть файл

@ -0,0 +1,93 @@
targetScope='subscription'
param location string = 'westus2'
param env string = 'dev'
param prefix string
param postfix string
param resourceGroupName string = 'rg-wus-test'
var baseName = '${prefix}${postfix}'
resource resgrp 'Microsoft.Resources/resourceGroups@2020-06-01' = {
name: resourceGroupName
location: location
}
// storage account
module stoacct './modules/stoacct.bicep' = {
name: 'stoacct'
scope: resourceGroup(resgrp.name)
params: {
env: env
baseName: baseName
location: location
}
}
// keyvault
module kv './modules/kv.bicep' = {
name: 'kv'
scope: resourceGroup(resgrp.name)
params:{
env: env
location: location
baseName: baseName
}
}
// appinsights
module appinsight './modules/appinsight.bicep' = {
name: 'appinsight'
scope: resourceGroup(resgrp.name)
params:{
baseName: baseName
env: env
location: location
}
}
// container registry
module cr './modules/cr.bicep' = {
name: 'cr'
scope: resourceGroup(resgrp.name)
params:{
baseName: baseName
env: env
location: location
}
}
// amls workspace
module amls './modules/amls.bicep' = {
name: 'amls'
scope: resourceGroup(resgrp.name)
params:{
baseName: baseName
env: env
location: location
stoacctid: stoacct.outputs.stoacctOut
kvid: kv.outputs.kvOut
appinsightid: appinsight.outputs.appinsightOut
crid: cr.outputs.crOut
}
}
// aml compute instance
module amlci './modules/amlcomputeinstance.bicep' = {
name: 'amlci'
scope: resourceGroup(resgrp.name)
params:{
baseName: baseName
env: env
location: location
workspaceName: amls.outputs.amlsName
}
}

516
bicep/main.json Normal file
Просмотреть файл

@ -0,0 +1,516 @@
{
"$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "11208633954998583577"
}
},
"parameters": {
"location": {
"type": "string",
"defaultValue": "westus2"
},
"env": {
"type": "string",
"defaultValue": "dev"
},
"prefix": {
"type": "string"
},
"postfix": {
"type": "string"
},
"resourceGroupName": {
"type": "string",
"defaultValue": "rg-wus-test"
}
},
"variables": {
"baseName": "[format('{0}{1}', parameters('prefix'), parameters('postfix'))]"
},
"resources": [
{
"type": "Microsoft.Resources/resourceGroups",
"apiVersion": "2020-06-01",
"name": "[parameters('resourceGroupName')]",
"location": "[parameters('location')]"
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "stoacct",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"env": {
"value": "[parameters('env')]"
},
"baseName": {
"value": "[variables('baseName')]"
},
"location": {
"value": "[parameters('location')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "13854706444404712543"
}
},
"parameters": {
"env": {
"type": "string"
},
"baseName": {
"type": "string"
},
"location": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2019-04-01",
"name": "[format('{0}{1}sa', parameters('env'), parameters('baseName'))]",
"location": "[parameters('location')]",
"sku": {
"name": "Standard_LRS"
},
"kind": "StorageV2",
"properties": {
"encryption": {
"services": {
"blob": {
"enabled": true
},
"file": {
"enabled": true
}
},
"keySource": "Microsoft.Storage"
},
"supportsHttpsTrafficOnly": true
}
}
],
"outputs": {
"stoacctOut": {
"type": "string",
"value": "[resourceId('Microsoft.Storage/storageAccounts', format('{0}{1}sa', parameters('env'), parameters('baseName')))]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "kv",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"env": {
"value": "[parameters('env')]"
},
"location": {
"value": "[parameters('location')]"
},
"baseName": {
"value": "[variables('baseName')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "3960831692549416869"
}
},
"parameters": {
"baseName": {
"type": "string"
},
"env": {
"type": "string"
},
"location": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.KeyVault/vaults",
"apiVersion": "2019-09-01",
"name": "[format('{0}-{1}-kv', parameters('env'), parameters('baseName'))]",
"location": "[parameters('location')]",
"properties": {
"tenantId": "[subscription().tenantId]",
"sku": {
"name": "standard",
"family": "A"
},
"accessPolicies": []
}
}
],
"outputs": {
"kvOut": {
"type": "string",
"value": "[resourceId('Microsoft.KeyVault/vaults', format('{0}-{1}-kv', parameters('env'), parameters('baseName')))]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "appinsight",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"baseName": {
"value": "[variables('baseName')]"
},
"env": {
"value": "[parameters('env')]"
},
"location": {
"value": "[parameters('location')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "2591061638125956638"
}
},
"parameters": {
"baseName": {
"type": "string"
},
"env": {
"type": "string"
},
"location": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.Insights/components",
"apiVersion": "2020-02-02-preview",
"name": "[format('{0}{1}-appin', parameters('env'), parameters('baseName'))]",
"location": "[parameters('location')]",
"kind": "web",
"properties": {
"Application_Type": "web"
}
}
],
"outputs": {
"appinsightOut": {
"type": "string",
"value": "[resourceId('Microsoft.Insights/components', format('{0}{1}-appin', parameters('env'), parameters('baseName')))]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "cr",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"baseName": {
"value": "[variables('baseName')]"
},
"env": {
"value": "[parameters('env')]"
},
"location": {
"value": "[parameters('location')]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "12155558635582316098"
}
},
"parameters": {
"env": {
"type": "string"
},
"baseName": {
"type": "string"
},
"location": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.ContainerRegistry/registries",
"apiVersion": "2020-11-01-preview",
"name": "[format('{0}{1}cr', parameters('env'), parameters('baseName'))]",
"location": "[parameters('location')]",
"sku": {
"name": "Standard"
},
"properties": {
"adminUserEnabled": true
}
}
],
"outputs": {
"crOut": {
"type": "string",
"value": "[resourceId('Microsoft.ContainerRegistry/registries', format('{0}{1}cr', parameters('env'), parameters('baseName')))]"
}
}
}
},
"dependsOn": [
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "amls",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"baseName": {
"value": "[variables('baseName')]"
},
"env": {
"value": "[parameters('env')]"
},
"location": {
"value": "[parameters('location')]"
},
"stoacctid": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'stoacct')).outputs.stoacctOut.value]"
},
"kvid": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'kv')).outputs.kvOut.value]"
},
"appinsightid": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'appinsight')).outputs.appinsightOut.value]"
},
"crid": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'cr')).outputs.crOut.value]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "18023230433604735324"
}
},
"parameters": {
"location": {
"type": "string"
},
"baseName": {
"type": "string"
},
"env": {
"type": "string"
},
"stoacctid": {
"type": "string"
},
"kvid": {
"type": "string"
},
"appinsightid": {
"type": "string"
},
"crid": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.MachineLearningServices/workspaces",
"apiVersion": "2020-09-01-preview",
"name": "[format('{0}{1}-ws', parameters('env'), parameters('baseName'))]",
"location": "[parameters('location')]",
"identity": {
"type": "SystemAssigned"
},
"sku": {
"tier": "basic",
"name": "basic"
},
"properties": {
"friendlyName": "[format('{0}{1}-ws', parameters('env'), parameters('baseName'))]",
"storageAccount": "[parameters('stoacctid')]",
"keyVault": "[parameters('kvid')]",
"applicationInsights": "[parameters('appinsightid')]",
"containerRegistry": "[parameters('crid')]",
"encryption": {
"status": "Disabled",
"keyVaultProperties": {
"keyIdentifier": "",
"keyVaultArmId": ""
}
}
}
}
],
"outputs": {
"amlsName": {
"type": "string",
"value": "[format('{0}{1}-ws', parameters('env'), parameters('baseName'))]"
}
}
}
},
"dependsOn": [
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'appinsight')]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'cr')]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'kv')]",
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]",
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'stoacct')]"
]
},
{
"type": "Microsoft.Resources/deployments",
"apiVersion": "2020-10-01",
"name": "amlci",
"resourceGroup": "[parameters('resourceGroupName')]",
"properties": {
"expressionEvaluationOptions": {
"scope": "inner"
},
"mode": "Incremental",
"parameters": {
"baseName": {
"value": "[variables('baseName')]"
},
"env": {
"value": "[parameters('env')]"
},
"location": {
"value": "[parameters('location')]"
},
"workspaceName": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'amls')).outputs.amlsName.value]"
}
},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"metadata": {
"_generator": {
"name": "bicep",
"version": "0.5.6.12127",
"templateHash": "2016431671585526523"
}
},
"parameters": {
"location": {
"type": "string"
},
"baseName": {
"type": "string"
},
"env": {
"type": "string"
},
"computeInstanceName": {
"type": "string",
"defaultValue": "[format('{0}-{1}-ci', parameters('env'), parameters('baseName'))]"
},
"workspaceName": {
"type": "string"
}
},
"resources": [
{
"type": "Microsoft.MachineLearningServices/workspaces/computes",
"apiVersion": "2020-09-01-preview",
"name": "[format('{0}/{1}', parameters('workspaceName'), parameters('computeInstanceName'))]",
"location": "[parameters('location')]",
"properties": {
"computeType": "AmlCompute",
"properties": {
"vmSize": "Standard_DS3_v2",
"subnet": "[json('null')]",
"osType": "Linux",
"scaleSettings": {
"maxNodeCount": 4,
"minNodeCount": 0
}
}
}
}
]
}
},
"dependsOn": [
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'amls')]",
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]"
]
}
]
}

25
bicep/modules/amlcomputeinstance.bicep Normal file
Просмотреть файл

@ -0,0 +1,25 @@
param location string
param baseName string
param env string
param computeInstanceName string = '${env}-${baseName}-ci'
param workspaceName string
resource amlci 'Microsoft.MachineLearningServices/workspaces/computes@2020-09-01-preview' = {
name: '${workspaceName}/${computeInstanceName}'
location: location
properties:{
computeType: 'AmlCompute'
properties:{
vmSize: 'Standard_DS3_v2'
subnet: json('null')
osType:'Linux'
scaleSettings:{
maxNodeCount: 4
minNodeCount: 0
}
}
}
}

39
bicep/modules/amls.bicep Normal file
Просмотреть файл

@ -0,0 +1,39 @@
param location string
param baseName string
param env string
param stoacctid string
param kvid string
param appinsightid string
param crid string
// azure machine learning service
resource amls 'Microsoft.MachineLearningServices/workspaces@2020-09-01-preview' = {
name: '${env}${baseName}-ws'
location: location
identity: {
type: 'SystemAssigned'
}
sku:{
tier: 'basic'
name: 'basic'
}
properties:{
friendlyName: '${env}${baseName}-ws'
storageAccount: stoacctid
keyVault: kvid
applicationInsights: appinsightid
containerRegistry: crid
encryption:{
status: 'Disabled'
keyVaultProperties:{
keyIdentifier: ''
keyVaultArmId: ''
}
}
}
}
output amlsName string = amls.name

16
bicep/modules/appinsight.bicep Normal file
Просмотреть файл

@ -0,0 +1,16 @@
param baseName string
param env string
param location string
// app insights
resource appinsight 'Microsoft.Insights/components@2020-02-02-preview' = {
name: '${env}${baseName}-appin'
location: location
kind: 'web'
properties:{
Application_Type: 'web'
}
}
output appinsightOut string = appinsight.id

17
bicep/modules/cr.bicep Normal file
Просмотреть файл

@ -0,0 +1,17 @@
param env string
param baseName string
param location string
resource cr 'Microsoft.ContainerRegistry/registries@2020-11-01-preview' = {
name: '${env}${baseName}cr'
location: location
sku: {
name: 'Standard'
}
properties:{
adminUserEnabled:true
}
}
output crOut string = cr.id

20
bicep/modules/kv.bicep Normal file
Просмотреть файл

@ -0,0 +1,20 @@
param baseName string
param env string
param location string
// keyvault
resource kv 'Microsoft.KeyVault/vaults@2019-09-01' = {
name: '${env}-${baseName}-kv'
location: location
properties:{
tenantId: subscription().tenantId
sku: {
name: 'standard'
family: 'A'
}
accessPolicies: []
}
}
output kvOut string = kv.id

30
bicep/modules/stoacct.bicep Normal file
Просмотреть файл

@ -0,0 +1,30 @@
param env string
param baseName string
param location string
// stroage account
resource stoacct 'Microsoft.Storage/storageAccounts@2019-04-01' = {
name: '${env}${baseName}sa'
location: location
sku:{
name:'Standard_LRS'
}
kind: 'StorageV2'
properties:{
encryption:{
services:{
blob:{
enabled: true
}
file:{
enabled: true
}
}
keySource: 'Microsoft.Storage'
}
supportsHttpsTrafficOnly: true
}
}
output stoacctOut string = stoacct.id

34
bicep/pipelines/bicep-iac-std-pipeline.yml Normal file
Просмотреть файл

@ -0,0 +1,34 @@
variables:
- template: ../../../config-aml.yml
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
# 'main' branch: PRD environment
- template: ../../../config-infra-prod.yml
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
# 'develop' or feature branches: DEV environment
- template: ../../../config-infra-dev.yml
trigger:
- none
pool:
vmImage: $(ap_vm_image)
stages :
- stage: CheckOutBicepAndDeploy
displayName: Deploy AML Workspace
jobs:
- job: DeployBicep
displayName: Create Bicep Deployment
steps:
- checkout: self
- task: AzureCLI@2
displayName: Running Deployment
inputs:
azureSubscription: $(ado_service_connection_rg)
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
az --version
echo "deploying bicep..."
az deployment sub create --name $(Build.DefinitionName) --location $(location) --template-file ./infrastructure/bicep/main.bicep --parameters location=$(location) resourceGroupName=$(resource_group) prefix=$(namespace) postfix=$(postfix)

94
terraform/aml_deploy.tf Normal file
Просмотреть файл

@ -0,0 +1,94 @@
# Resource group
module "resource_group" {
source = "./modules/resource-group"
location = var.location
prefix = var.prefix
postfix = var.postfix
tags = local.tags
}
# Azure Machine Learning workspace
module "aml_workspace" {
source = "./modules/aml-workspace"
rg_name = module.resource_group.name
location = module.resource_group.location
prefix = var.prefix
postfix = var.postfix
storage_account_id = module.storage_account_aml.id
key_vault_id = module.key_vault.id
application_insights_id = module.application_insights.id
container_registry_id = module.container_registry.id
enable_aml_computecluster = var.enable_aml_computecluster
storage_account_name = module.storage_account_aml.name
tags = local.tags
}
# Storage account
module "storage_account_aml" {
source = "./modules/storage-account"
rg_name = module.resource_group.name
location = module.resource_group.location
prefix = var.prefix
postfix = "${var.postfix}aml"
hns_enabled = false
firewall_bypass = ["AzureServices"]
firewall_virtual_network_subnet_ids = []
tags = local.tags
}
# Key vault
module "key_vault" {
source = "./modules/key-vault"
rg_name = module.resource_group.name
location = module.resource_group.location
prefix = var.prefix
postfix = var.postfix
tags = local.tags
}
# Application insights
module "application_insights" {
source = "./modules/application-insights"
rg_name = module.resource_group.name
location = module.resource_group.location
prefix = var.prefix
postfix = var.postfix
tags = local.tags
}
# Container registry
module "container_registry" {
source = "./modules/container-registry"
rg_name = module.resource_group.name
location = module.resource_group.location
prefix = var.prefix
postfix = var.postfix
tags = local.tags
}

9
terraform/locals.tf Normal file
Просмотреть файл

@ -0,0 +1,9 @@
locals {
tags = {
Owner = "mlops-tabular"
Project = "mlops-tabular"
Environment = "${var.environment}"
Toolkit = "Terraform"
Name = "${var.prefix}"
}
}

18
terraform/main.tf Normal file
Просмотреть файл

@ -0,0 +1,18 @@
terraform {
backend "azurerm" {}
required_providers {
azurerm = {
version = "= 2.99.0"
}
}
}
provider "azurerm" {
features {}
}
data "azurerm_client_config" "current" {}
data "http" "ip" {
url = "https://ifconfig.me"
}

104
terraform/modules/aml-workspace/main.tf Normal file
Просмотреть файл

@ -0,0 +1,104 @@
resource "azurerm_machine_learning_workspace" "adl_mlw" {
name = "mlw-${var.prefix}-${var.postfix}"
location = var.location
resource_group_name = var.rg_name
application_insights_id = var.application_insights_id
key_vault_id = var.key_vault_id
storage_account_id = var.storage_account_id
container_registry_id = var.container_registry_id
identity {
type = "SystemAssigned"
}
tags = var.tags
}
# Compute cluster
resource "azurerm_machine_learning_compute_cluster" "adl_aml_ws_compute_cluster" {
name = "mlwcc${var.prefix}${var.postfix}"
location = var.location
vm_priority = "LowPriority"
vm_size = "STANDARD_DS2_V2"
machine_learning_workspace_id = azurerm_machine_learning_workspace.adl_mlw.id
count = var.enable_aml_computecluster ? 1 : 0
scale_settings {
min_node_count = 0
max_node_count = 1
scale_down_nodes_after_idle_duration = "PT120S" # 120 seconds
}
identity {
type = "SystemAssigned"
}
}
# Datastore
resource "azurerm_resource_group_template_deployment" "arm_aml_create_datastore" {
name = "arm_aml_create_datastore"
resource_group_name = var.rg_name
deployment_mode = "Incremental"
parameters_content = jsonencode({
"WorkspaceName" = {
value = azurerm_machine_learning_workspace.adl_mlw.name
},
"StorageAccountName" = {
value = var.storage_account_name
}
})
depends_on = [time_sleep.wait_30_seconds]
template_content = <<TEMPLATE
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"WorkspaceName": {
"type": "String"
},
"StorageAccountName": {
"type": "String"
}
},
"resources": [
{
"type": "Microsoft.MachineLearningServices/workspaces/datastores",
"apiVersion": "2021-03-01-preview",
"name": "[concat(parameters('WorkspaceName'), '/default')]",
"dependsOn": [],
"properties": {
"contents": {
"accountName": "[parameters('StorageAccountName')]",
"containerName": "default",
"contentsType": "AzureBlob",
"credentials": {
"credentialsType": "None"
},
"endpoint": "core.windows.net",
"protocol": "https"
},
"description": "Default datastore for mlops-tabular",
"isDefault": false,
"properties": {
"ServiceDataAccessAuthIdentity": "None"
},
"tags": {}
}
}
]
}
TEMPLATE
}
resource "time_sleep" "wait_30_seconds" {
depends_on = [
azurerm_machine_learning_workspace.adl_mlw
]
create_duration = "30s"
}

0
terraform/modules/aml-workspace/outputs.tf Normal file
Просмотреть файл

55
terraform/modules/aml-workspace/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,55 @@
variable "rg_name" {
type = string
description = "Resource group name"
}
variable "location" {
type = string
description = "Location of the resource group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the deployed resource"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}
variable "storage_account_id" {
type = string
description = "The ID of the Storage Account linked to AML workspace"
}
variable "key_vault_id" {
type = string
description = "The ID of the Key Vault linked to AML workspace"
}
variable "application_insights_id" {
type = string
description = "The ID of the Application Insights linked to AML workspace"
}
variable "container_registry_id" {
type = string
description = "The ID of the Container Registry linked to AML workspace"
}
variable "enable_aml_computecluster" {
description = "Variable to enable or disable AML compute cluster"
default = false
}
variable "storage_account_name" {
type = string
description = "The Name of the Storage Account linked to AML workspace"
}

8
terraform/modules/application-insights/main.tf Normal file
Просмотреть файл

@ -0,0 +1,8 @@
resource "azurerm_application_insights" "adl_appi" {
name = "appi-${var.prefix}-${var.postfix}"
location = var.location
resource_group_name = var.rg_name
application_type = "web"
tags = var.tags
}

3
terraform/modules/application-insights/outputs.tf Normal file
Просмотреть файл

@ -0,0 +1,3 @@
output "id" {
value = azurerm_application_insights.adl_appi.id
}

25
terraform/modules/application-insights/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,25 @@
variable "rg_name" {
type = string
description = "Resource group name"
}
variable "location" {
type = string
description = "Location of the resource group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the deployed resource"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}

20
terraform/modules/container-registry/main.tf Normal file
Просмотреть файл

@ -0,0 +1,20 @@
locals {
safe_prefix = replace(var.prefix, "-", "")
safe_postfix = replace(var.postfix, "-", "")
}
resource "azurerm_container_registry" "adl_cr" {
name = "cr${local.safe_prefix}${local.safe_postfix}"
resource_group_name = var.rg_name
location = var.location
sku = "Premium"
admin_enabled = false
network_rule_set {
default_action = "Deny"
ip_rule = []
virtual_network = []
}
tags = var.tags
}

3
terraform/modules/container-registry/outputs.tf Normal file
Просмотреть файл

@ -0,0 +1,3 @@
output "id" {
value = azurerm_container_registry.adl_cr.id
}

25
terraform/modules/container-registry/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,25 @@
variable "rg_name" {
type = string
description = "Resource group name"
}
variable "location" {
type = string
description = "Location of the resource group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the deployed resource"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}

18
terraform/modules/key-vault/main.tf Normal file
Просмотреть файл

@ -0,0 +1,18 @@
data "azurerm_client_config" "current" {}
resource "azurerm_key_vault" "adl_kv" {
name = "kv-${var.prefix}-${var.postfix}"
location = var.location
resource_group_name = var.rg_name
tenant_id = data.azurerm_client_config.current.tenant_id
sku_name = "standard"
network_acls {
default_action = "Deny"
ip_rules = []
virtual_network_subnet_ids = []
bypass = "None"
}
tags = var.tags
}

7
terraform/modules/key-vault/outputs.tf Normal file
Просмотреть файл

@ -0,0 +1,7 @@
output "id" {
value = azurerm_key_vault.adl_kv.id
}
output "name" {
value = azurerm_key_vault.adl_kv.name
}

25
terraform/modules/key-vault/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,25 @@
variable "rg_name" {
type = string
description = "Resource group name"
}
variable "location" {
type = string
description = "Location of the resource group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the deployed resource"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}

5
terraform/modules/resource-group/main.tf Normal file
Просмотреть файл

@ -0,0 +1,5 @@
resource "azurerm_resource_group" "adl_rg" {
name = "rg-${var.prefix}-${var.postfix}"
location = var.location
tags = var.tags
}

7
terraform/modules/resource-group/outputs.tf Normal file
Просмотреть файл

@ -0,0 +1,7 @@
output "name" {
value = azurerm_resource_group.adl_rg.name
}
output "location" {
value = azurerm_resource_group.adl_rg.location
}

21
terraform/modules/resource-group/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,21 @@
variable "location" {
type = string
default = "North Europe"
description = "Location of the Resource Group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the Resource Group"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}

34
terraform/modules/storage-account/main.tf Normal file
Просмотреть файл

@ -0,0 +1,34 @@
data "azurerm_client_config" "current" {}
data "http" "ip" {
url = "https://ifconfig.me"
}
locals {
safe_prefix = replace(var.prefix, "-", "")
safe_postfix = replace(var.postfix, "-", "")
}
resource "azurerm_storage_account" "adl_st" {
name = "st${local.safe_prefix}${local.safe_postfix}"
resource_group_name = var.rg_name
location = var.location
account_tier = "Standard"
account_replication_type = "LRS"
account_kind = "StorageV2"
is_hns_enabled = var.hns_enabled
tags = var.tags
}
# Virtual Network & Firewall configuration
resource "azurerm_storage_account_network_rules" "firewall_rules" {
resource_group_name = var.rg_name
storage_account_name = azurerm_storage_account.adl_st.name
default_action = "Allow"
ip_rules = [] # [data.http.ip.body]
virtual_network_subnet_ids = var.firewall_virtual_network_subnet_ids
bypass = var.firewall_bypass
}

7
terraform/modules/storage-account/outputs.tf Normal file
Просмотреть файл

@ -0,0 +1,7 @@
output "id" {
value = azurerm_storage_account.adl_st.id
}
output "name" {
value = azurerm_storage_account.adl_st.name
}

39
terraform/modules/storage-account/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,39 @@
variable "rg_name" {
type = string
description = "Resource group name"
}
variable "location" {
type = string
description = "Location of the resource group"
}
variable "tags" {
type = map(string)
default = {}
description = "A mapping of tags which should be assigned to the Resource Group"
}
variable "prefix" {
type = string
description = "Prefix for the module name"
}
variable "postfix" {
type = string
description = "Postfix for the module name"
}
variable "hns_enabled" {
type = bool
description = "Hierarchical namespaces enabled/disabled"
default = true
}
variable "firewall_virtual_network_subnet_ids" {
default = []
}
variable "firewall_bypass" {
default = ["None"]
}

55
terraform/pipelines/tf-ado-deploy-infra.yml Normal file
Просмотреть файл

@ -0,0 +1,55 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
variables:
- template: ../../../config-aml.yml
- ${{ if eq(variables['Build.SourceBranchName'], 'main') }}:
# 'main' branch: PRD environment
- template: ../../../config-infra-prod.yml
- ${{ if ne(variables['Build.SourceBranchName'], 'main') }}:
# 'develop' or feature branches: DEV environment
- template: ../../../config-infra-dev.yml
trigger:
- none
pool:
vmImage: $(ap_vm_image)
resources:
repositories:
- repository: mlops-templates
name: Azure/mlops-templates
endpoint: mlops-v2-tabular
type: github
ref: main #branch name
stages :
- stage: CreateStorageAccountForTerraformState
displayName: Create Storage for Terraform
jobs:
- job: CreateStorageForTerraform
displayName: Create Storage for Terraform
steps:
- checkout: self
path: s/
- checkout: mlops-templates
path: s/templates/
- template: templates/infra/create-resource-group.yml@mlops-templates
- template: templates/infra/create-storage-account.yml@mlops-templates
- template: templates/infra/create-storage-container.yml@mlops-templates
- stage: DeployAzureMachineLearningRG
displayName: Deploy AML Resource Group
jobs:
- job: DeployAMLWorkspace
displayName: 'Deploy AML Workspace'
steps:
- checkout: self
path: s/
- checkout: mlops-templates
path: s/templates/
- template: templates/infra/install-terraform.yml@mlops-templates
- template: templates/infra/run-terraform-init.yml@mlops-templates
- template: templates/infra/run-terraform-validate.yml@mlops-templates
- template: templates/infra/run-terraform-plan.yml@mlops-templates
- template: templates/infra/run-terraform-apply.yml@mlops-templates

23
terraform/variables.tf Normal file
Просмотреть файл

@ -0,0 +1,23 @@
variable "location" {
type = string
description = "Location of the resource group and modules"
}
variable "prefix" {
type = string
description = "Prefix for module names"
}
variable "environment" {
type = string
description = "Environment information"
}
variable "postfix" {
type = string
description = "Postfix for module names"
}
variable "enable_aml_computecluster" {
description = "Variable to enable or disable AML compute cluster"
}