4074 строки
204 KiB
JSON
4074 строки
204 KiB
JSON
{
|
|
"$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "13503021382899524313"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"prefix": {
|
|
"type": "string",
|
|
"defaultValue": "sri",
|
|
"metadata": {
|
|
"description": "2-4 chars to prefix the Azure resources, NOTE: no number or symbols"
|
|
},
|
|
"maxLength": 4,
|
|
"minLength": 2
|
|
},
|
|
"adminUsername": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Client PC username, NOTE: do not use admin"
|
|
}
|
|
},
|
|
"adminPassword": {
|
|
"type": "secureString",
|
|
"minLength": 8,
|
|
"metadata": {
|
|
"description": "Client PC password, with atleast 8 char length containing uppercase, digits and special characters "
|
|
}
|
|
},
|
|
"linkAkstoAml": {
|
|
"type": "bool",
|
|
"defaultValue": true
|
|
},
|
|
"deployADBCluster": {
|
|
"type": "bool",
|
|
"defaultValue": true
|
|
},
|
|
"updateAKVKeys": {
|
|
"type": "bool",
|
|
"defaultValue": true
|
|
},
|
|
"adb_pat_lifetime": {
|
|
"type": "string",
|
|
"defaultValue": "3600"
|
|
},
|
|
"adb_cluster_name": {
|
|
"type": "string",
|
|
"defaultValue": "test-cluster-01"
|
|
},
|
|
"adb_spark_version": {
|
|
"type": "string",
|
|
"defaultValue": "7.3.x-scala2.12"
|
|
},
|
|
"adb_node_type": {
|
|
"type": "string",
|
|
"defaultValue": "Standard_D3_v2"
|
|
},
|
|
"adb_num_worker": {
|
|
"type": "string",
|
|
"defaultValue": "3"
|
|
},
|
|
"adb_auto_terminate_min": {
|
|
"type": "string",
|
|
"defaultValue": "30"
|
|
},
|
|
"aksAgentCount": {
|
|
"type": "int",
|
|
"defaultValue": 3,
|
|
"maxValue": 50,
|
|
"minValue": 3,
|
|
"metadata": {
|
|
"description": "The number of nodes for the cluster."
|
|
}
|
|
},
|
|
"aksAgentVMSize": {
|
|
"type": "string",
|
|
"defaultValue": "Standard_A4_v2",
|
|
"metadata": {
|
|
"description": "The size of the VM instances"
|
|
}
|
|
},
|
|
"location": {
|
|
"type": "string",
|
|
"defaultValue": "southeastasia",
|
|
"metadata": {
|
|
"description": "Default location of the resources"
|
|
}
|
|
},
|
|
"hubVnetName": {
|
|
"type": "string",
|
|
"defaultValue": "hubvnet",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"spokeVnetName": {
|
|
"type": "string",
|
|
"defaultValue": "spokevnet",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"HubVnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.0.0.0/16",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"FirewallSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.0.1.0/26",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"clientDevicesSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.0.200.0/24",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"SpokeVnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.179.0.0/16",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"PrivateSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.179.0.0/18",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"PublicSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.179.64.0/18",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"AksSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.179.128.0/18",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"PrivateLinkSubnetCidr": {
|
|
"type": "string",
|
|
"defaultValue": "10.179.192.0/18",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"webappDestinationAddresses": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"52.187.145.107/32",
|
|
"52.187.0.85/32"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia ADB webapp address"
|
|
}
|
|
},
|
|
"logBlobstorageDomains": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"[format('dblogprodseasia.blob.{0}', environment().suffixes.storage)]"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia ADB log blob"
|
|
}
|
|
},
|
|
"extendedInfraIp": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"20.195.104.64/28"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia ADB extended ip"
|
|
}
|
|
},
|
|
"sccReplayDomain": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"tunnel.southeastasia.azuredatabricks.net"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia SCC relay Domain"
|
|
}
|
|
},
|
|
"metastoreDomains": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"consolidated-southeastasia-prod-metastore.mysql.database.azure.com"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia SDB metastore"
|
|
}
|
|
},
|
|
"eventHubEndpointDomain": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"prod-southeastasia-observabilityeventhubs.servicebus.windows.net"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia EventHub endpoint"
|
|
}
|
|
},
|
|
"artifactBlobStoragePrimaryDomains": {
|
|
"type": "array",
|
|
"defaultValue": [
|
|
"[format('dbartifactsprodseap.blob.{0}', environment().suffixes.storage)]",
|
|
"[format('arprodseapa1.blob.{0}', environment().suffixes.storage)]",
|
|
"[format('arprodseapa2.blob.{0}', environment().suffixes.storage)]",
|
|
"[format('arprodseapa3.blob.{0}', environment().suffixes.storage)]",
|
|
"[format('dbartifactsprodeap.blob.{0}', environment().suffixes.storage)]"
|
|
],
|
|
"metadata": {
|
|
"description": "Southeastasia Artifacts Blob"
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"uniqueSubString": "[uniqueString(guid(subscription().subscriptionId))]",
|
|
"uString": "[format('{0}{1}', parameters('prefix'), variables('uniqueSubString'))]",
|
|
"storageAccountName": "[format('{0}stg01', substring(variables('uString'), 0, 10))]",
|
|
"keyVaultName": "[format('{0}-akv-00', substring(variables('uString'), 0, 6))]",
|
|
"resourceGroupName": "[format('{0}-rg', substring(variables('uString'), 0, 6))]",
|
|
"adbWorkspaceName": "[format('{0}-AdbWksp', substring(variables('uString'), 0, 6))]",
|
|
"nsgName": "[format('{0}-nsg', substring(variables('uString'), 0, 6))]",
|
|
"firewallName": "[format('{0}-HubFW', substring(variables('uString'), 0, 6))]",
|
|
"firewallPublicIpName": "[format('{0}-FWPIp', substring(variables('uString'), 0, 6))]",
|
|
"fwRoutingTable": "[format('{0}-AdbRoutingTbl', substring(variables('uString'), 0, 6))]",
|
|
"clientPcName": "[format('{0}-ClientPc', substring(variables('uString'), 0, 6))]",
|
|
"eHNameSpace": "[format('{0}-eh', substring(variables('uString'), 0, 6))]",
|
|
"adbAkvLinkName": "[format('{0}SecretScope', substring(variables('uString'), 0, 6))]",
|
|
"amlWorkspaceName": "[format('{0}-AmlWksp', substring(variables('uString'), 0, 6))]",
|
|
"containerRegistryName": "[format('{0}registry', substring(variables('uString'), 0, 6))]",
|
|
"applicationInsightsName": "[format('{0}-AppInsights', substring(variables('uString'), 0, 6))]",
|
|
"sslLeafName": "[toLower(substring(variables('uString'), 0, 6))]",
|
|
"aksAmlComputeName": "[format('aks-{0}', substring(variables('uString'), 0, 6))]",
|
|
"aksDNSPrefix": "[toLower(substring(variables('uString'), 0, 6))]",
|
|
"eventHubName": "[variables('eHNameSpace')]",
|
|
"managedIdentityName": "[format('{0}Identity', substring(variables('uString'), 0, 6))]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Resources/resourceGroups",
|
|
"apiVersion": "2021-04-01",
|
|
"name": "[variables('resourceGroupName')]",
|
|
"location": "[parameters('location')]"
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "ManagedIdentity",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"managedIdentityName": {
|
|
"value": "[variables('managedIdentityName')]"
|
|
},
|
|
"location": {
|
|
"value": "[parameters('location')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "2361919279329234047"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"location": {
|
|
"type": "string"
|
|
},
|
|
"managedIdentityName": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"ownerRoleDefId": "8e3af657-a8ff-443c-a75c-2fe8c4bcb635"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.ManagedIdentity/userAssignedIdentities",
|
|
"apiVersion": "2018-11-30",
|
|
"name": "[parameters('managedIdentityName')]",
|
|
"location": "[parameters('location')]"
|
|
},
|
|
{
|
|
"type": "Microsoft.Authorization/roleAssignments",
|
|
"apiVersion": "2020-08-01-preview",
|
|
"name": "[guid(variables('ownerRoleDefId'), resourceGroup().id)]",
|
|
"properties": {
|
|
"principalType": "ServicePrincipal",
|
|
"principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))).principalId]",
|
|
"roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('ownerRoleDefId'))]"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"mIdentityId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))]"
|
|
},
|
|
"mIdentityClientId": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', parameters('managedIdentityName'))).clientId]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "NetworkSecurityGroup",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"securityGroupName": {
|
|
"value": "[variables('nsgName')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "71359721707976321"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"securityGroupName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "The name of the network security group to create."
|
|
}
|
|
},
|
|
"securityGroupLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Location for all resources."
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/networkSecurityGroups",
|
|
"apiVersion": "2020-05-01",
|
|
"name": "[parameters('securityGroupName')]",
|
|
"location": "[parameters('securityGroupLocation')]",
|
|
"properties": {
|
|
"securityRules": [
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-inbound",
|
|
"properties": {
|
|
"description": "Required for worker nodes communication within a cluster.",
|
|
"protocol": "*",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "*",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "VirtualNetwork",
|
|
"access": "Allow",
|
|
"priority": 100,
|
|
"direction": "Inbound"
|
|
}
|
|
},
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-databricks-webapp",
|
|
"properties": {
|
|
"description": "Required for workers communication with Databricks Webapp.",
|
|
"protocol": "Tcp",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "443",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "AzureDatabricks",
|
|
"access": "Allow",
|
|
"priority": 100,
|
|
"direction": "Outbound"
|
|
}
|
|
},
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-sql",
|
|
"properties": {
|
|
"description": "Required for workers communication with Azure SQL services.",
|
|
"protocol": "Tcp",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "3306",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "Sql",
|
|
"access": "Allow",
|
|
"priority": 101,
|
|
"direction": "Outbound"
|
|
}
|
|
},
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-storage",
|
|
"properties": {
|
|
"description": "Required for workers communication with Azure Storage services.",
|
|
"protocol": "Tcp",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "443",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "Storage",
|
|
"access": "Allow",
|
|
"priority": 102,
|
|
"direction": "Outbound"
|
|
}
|
|
},
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-worker-outbound",
|
|
"properties": {
|
|
"description": "Required for worker nodes communication within a cluster.",
|
|
"protocol": "*",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "*",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "VirtualNetwork",
|
|
"access": "Allow",
|
|
"priority": 103,
|
|
"direction": "Outbound"
|
|
}
|
|
},
|
|
{
|
|
"name": "Microsoft.Databricks-workspaces_UseOnly_databricks-worker-to-eventhub",
|
|
"properties": {
|
|
"description": "Required for worker communication with Azure Eventhub services.",
|
|
"protocol": "Tcp",
|
|
"sourcePortRange": "*",
|
|
"destinationPortRange": "9093",
|
|
"sourceAddressPrefix": "VirtualNetwork",
|
|
"destinationAddressPrefix": "EventHub",
|
|
"access": "Allow",
|
|
"priority": 104,
|
|
"direction": "Outbound"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"outputs": {
|
|
"nsgName": {
|
|
"type": "string",
|
|
"value": "[parameters('securityGroupName')]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "RouteTable",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"routeTableName": {
|
|
"value": "[variables('fwRoutingTable')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "15357105487262340350"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"routeTableLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Azure datacentre Location to deploy the Firewall and IP Address"
|
|
}
|
|
},
|
|
"routeTableName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the Routing Table"
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/routeTables",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[parameters('routeTableName')]",
|
|
"location": "[parameters('routeTableLocation')]",
|
|
"properties": {
|
|
"disableBgpRoutePropagation": false
|
|
}
|
|
}
|
|
],
|
|
"outputs": {
|
|
"routeTblName": {
|
|
"type": "string",
|
|
"value": "[parameters('routeTableName')]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "HubandSpokeVnets",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"hubVnetName": {
|
|
"value": "[parameters('hubVnetName')]"
|
|
},
|
|
"spokeVnetName": {
|
|
"value": "[parameters('spokeVnetName')]"
|
|
},
|
|
"routeTableName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'RouteTable'), '2019-10-01').outputs.routeTblName.value]"
|
|
},
|
|
"securityGroupName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'NetworkSecurityGroup'), '2019-10-01').outputs.nsgName.value]"
|
|
},
|
|
"firewallSubnetCidr": {
|
|
"value": "[parameters('FirewallSubnetCidr')]"
|
|
},
|
|
"hubVnetCidr": {
|
|
"value": "[parameters('HubVnetCidr')]"
|
|
},
|
|
"spokeVnetCidr": {
|
|
"value": "[parameters('SpokeVnetCidr')]"
|
|
},
|
|
"publicSubnetCidr": {
|
|
"value": "[parameters('PublicSubnetCidr')]"
|
|
},
|
|
"privateSubnetCidr": {
|
|
"value": "[parameters('PrivateSubnetCidr')]"
|
|
},
|
|
"privatelinkSubnetCidr": {
|
|
"value": "[parameters('PrivateLinkSubnetCidr')]"
|
|
},
|
|
"clinetDevicesSubnetCidr": {
|
|
"value": "[parameters('clientDevicesSubnetCidr')]"
|
|
},
|
|
"AksSubnetCidr": {
|
|
"value": "[parameters('AksSubnetCidr')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "14348728657561399441"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"securityGroupName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "The name of the existing network security group to create."
|
|
}
|
|
},
|
|
"spokeVnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "The name of the virtual network to create."
|
|
}
|
|
},
|
|
"hubVnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "The name of the virtual network to create."
|
|
}
|
|
},
|
|
"privateSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "private-subnet",
|
|
"metadata": {
|
|
"description": "The name of the private subnet to create."
|
|
}
|
|
},
|
|
"privatelinkSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "privatelink-subnet",
|
|
"metadata": {
|
|
"description": "The name of the private subnet to create."
|
|
}
|
|
},
|
|
"publicSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "public-subnet",
|
|
"metadata": {
|
|
"description": "The name of the public subnet to create."
|
|
}
|
|
},
|
|
"firewallSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "AzureFirewallSubnet",
|
|
"metadata": {
|
|
"description": "The name of the firewall subnet to create."
|
|
}
|
|
},
|
|
"routeTableName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the Routing Table"
|
|
}
|
|
},
|
|
"vnetLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Location for all resources."
|
|
}
|
|
},
|
|
"spokeVnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the spoke vnet."
|
|
}
|
|
},
|
|
"hubVnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the hub vnet."
|
|
}
|
|
},
|
|
"privateSubnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the private subnet."
|
|
}
|
|
},
|
|
"publicSubnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the public subnet."
|
|
}
|
|
},
|
|
"firewallSubnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the firewall subnet."
|
|
}
|
|
},
|
|
"privatelinkSubnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the private link subnet.."
|
|
}
|
|
},
|
|
"aksSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "aks-subnet",
|
|
"metadata": {
|
|
"description": "The name of the private subnet to create."
|
|
}
|
|
},
|
|
"AksSubnetCidr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Cidr range for the AKS subnet.."
|
|
}
|
|
},
|
|
"clinetDevicesSubnetCidr": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"securityGroupId": "[resourceId('Microsoft.Network/networkSecurityGroups', parameters('securityGroupName'))]",
|
|
"serviceEndpoints": [
|
|
{
|
|
"service": "Microsoft.Storage",
|
|
"locations": [
|
|
"[resourceGroup().location]"
|
|
]
|
|
},
|
|
{
|
|
"service": "Microsoft.ContainerRegistry",
|
|
"locations": [
|
|
"[resourceGroup().location]"
|
|
]
|
|
},
|
|
{
|
|
"service": "Microsoft.KeyVault",
|
|
"locations": [
|
|
"[resourceGroup().location]"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/virtualNetworks",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[parameters('hubVnetName')]",
|
|
"location": "[parameters('vnetLocation')]",
|
|
"properties": {
|
|
"addressSpace": {
|
|
"addressPrefixes": [
|
|
"[parameters('hubVnetCidr')]"
|
|
]
|
|
},
|
|
"subnets": [
|
|
{
|
|
"name": "[parameters('firewallSubnetName')]",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('firewallSubnetCidr')]",
|
|
"privateEndpointNetworkPolicies": "Enabled",
|
|
"privateLinkServiceNetworkPolicies": "Enabled"
|
|
}
|
|
},
|
|
{
|
|
"name": "ClientDevices",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('clinetDevicesSubnetCidr')]",
|
|
"routeTable": {
|
|
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"enableDdosProtection": false
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[format('{0}/{1}', parameters('hubVnetName'), 'Peer-HubSpoke')]",
|
|
"properties": {
|
|
"allowVirtualNetworkAccess": true,
|
|
"allowForwardedTraffic": true,
|
|
"allowGatewayTransit": false,
|
|
"useRemoteGateways": false,
|
|
"remoteVirtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]",
|
|
"[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/virtualNetworks",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[parameters('spokeVnetName')]",
|
|
"location": "[parameters('vnetLocation')]",
|
|
"properties": {
|
|
"addressSpace": {
|
|
"addressPrefixes": [
|
|
"[parameters('spokeVnetCidr')]"
|
|
]
|
|
},
|
|
"subnets": [
|
|
{
|
|
"name": "[parameters('publicSubnetName')]",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('publicSubnetCidr')]",
|
|
"networkSecurityGroup": {
|
|
"id": "[variables('securityGroupId')]"
|
|
},
|
|
"routeTable": {
|
|
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
|
|
},
|
|
"serviceEndpoints": "[variables('serviceEndpoints')]",
|
|
"delegations": [
|
|
{
|
|
"name": "databricks-del-public",
|
|
"properties": {
|
|
"serviceName": "Microsoft.Databricks/workspaces"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "[parameters('privateSubnetName')]",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('privateSubnetCidr')]",
|
|
"networkSecurityGroup": {
|
|
"id": "[variables('securityGroupId')]"
|
|
},
|
|
"routeTable": {
|
|
"id": "[resourceId('Microsoft.Network/routeTables', parameters('routeTableName'))]"
|
|
},
|
|
"delegations": [
|
|
{
|
|
"name": "databricks-del-private",
|
|
"properties": {
|
|
"serviceName": "Microsoft.Databricks/workspaces"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "[parameters('privatelinkSubnetName')]",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('privatelinkSubnetCidr')]",
|
|
"privateEndpointNetworkPolicies": "Disabled",
|
|
"privateLinkServiceNetworkPolicies": "Enabled",
|
|
"serviceEndpoints": "[variables('serviceEndpoints')]"
|
|
}
|
|
},
|
|
{
|
|
"name": "[parameters('aksSubnetName')]",
|
|
"properties": {
|
|
"addressPrefix": "[parameters('AksSubnetCidr')]",
|
|
"serviceEndpoints": "[variables('serviceEndpoints')]"
|
|
}
|
|
}
|
|
],
|
|
"enableDdosProtection": false
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/virtualNetworks/virtualNetworkPeerings",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[format('{0}/{1}', parameters('spokeVnetName'), 'Peer-SpokeHub')]",
|
|
"properties": {
|
|
"allowVirtualNetworkAccess": true,
|
|
"allowForwardedTraffic": true,
|
|
"allowGatewayTransit": false,
|
|
"useRemoteGateways": false,
|
|
"remoteVirtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]"
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/virtualNetworks', parameters('hubVnetName'))]",
|
|
"[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"privatelinksubnet_id": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('spokeVnetName'), parameters('privatelinkSubnetName'))]"
|
|
},
|
|
"databricksPublicSubnetId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('spokeVnetName'), parameters('publicSubnetName'))]"
|
|
},
|
|
"spokeVnetName": {
|
|
"type": "string",
|
|
"value": "[parameters('spokeVnetName')]"
|
|
},
|
|
"hubVnetName": {
|
|
"type": "string",
|
|
"value": "[parameters('hubVnetName')]"
|
|
},
|
|
"aksSubnet_id": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('spokeVnetName'), parameters('aksSubnetName'))]"
|
|
},
|
|
"spokeVnetId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Network/virtualNetworks', parameters('spokeVnetName'))]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'NetworkSecurityGroup')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'RouteTable')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "StorageAccount",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"storageAccountName": {
|
|
"value": "[variables('storageAccountName')]"
|
|
},
|
|
"databricksPublicSubnetId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.databricksPublicSubnetId.value]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "1137679761133512415"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"storageAccountName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the storage account"
|
|
},
|
|
"maxLength": 24,
|
|
"minLength": 3
|
|
},
|
|
"storageContainerName": {
|
|
"type": "string",
|
|
"defaultValue": "data"
|
|
},
|
|
"databricksPublicSubnetId": {
|
|
"type": "string"
|
|
},
|
|
"storageAccountSku": {
|
|
"type": "string",
|
|
"defaultValue": "Standard_LRS",
|
|
"metadata": {
|
|
"description": "Storage Account Sku"
|
|
},
|
|
"allowedValues": [
|
|
"Standard_LRS",
|
|
"Standard_GRS",
|
|
"Standard_ZRS",
|
|
"Premium_LRS"
|
|
]
|
|
},
|
|
"encryptionEnabled": {
|
|
"type": "bool",
|
|
"defaultValue": true,
|
|
"metadata": {
|
|
"description": "Enable or disable Blob encryption at Rest."
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"location": "[resourceGroup().location]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Storage/storageAccounts",
|
|
"apiVersion": "2019-06-01",
|
|
"name": "[parameters('storageAccountName')]",
|
|
"tags": {
|
|
"displayName": "[parameters('storageAccountName')]",
|
|
"type": "Storage"
|
|
},
|
|
"location": "[variables('location')]",
|
|
"kind": "StorageV2",
|
|
"properties": {
|
|
"isHnsEnabled": true,
|
|
"minimumTlsVersion": "TLS1_2",
|
|
"supportsHttpsTrafficOnly": true,
|
|
"accessTier": "Hot",
|
|
"networkAcls": {
|
|
"bypass": "AzureServices",
|
|
"virtualNetworkRules": [
|
|
{
|
|
"id": "[parameters('databricksPublicSubnetId')]",
|
|
"action": "Allow",
|
|
"state": "succeeded"
|
|
}
|
|
],
|
|
"ipRules": [],
|
|
"defaultAction": "Deny"
|
|
},
|
|
"encryption": {
|
|
"keySource": "Microsoft.Storage",
|
|
"services": {
|
|
"blob": {
|
|
"enabled": "[parameters('encryptionEnabled')]"
|
|
},
|
|
"file": {
|
|
"enabled": "[parameters('encryptionEnabled')]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"sku": {
|
|
"name": "[parameters('storageAccountSku')]"
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
|
|
"apiVersion": "2019-06-01",
|
|
"name": "[format('{0}/default/{1}', parameters('storageAccountName'), parameters('storageContainerName'))]",
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"key1": {
|
|
"type": "string",
|
|
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-04-01').keys[0].value]"
|
|
},
|
|
"key2": {
|
|
"type": "string",
|
|
"value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-04-01').keys[1].value]"
|
|
},
|
|
"storageaccount_id": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageAccountName'))]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "DatabricksWorkspace",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"vnetName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.spokeVnetName.value]"
|
|
},
|
|
"adbWorkspaceSkuTier": {
|
|
"value": "premium"
|
|
},
|
|
"adbWorkspaceName": {
|
|
"value": "[variables('adbWorkspaceName')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "9573554296807436207"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"vnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"adbWorkspaceSkuTier": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
},
|
|
"allowedValues": [
|
|
"standard",
|
|
"premium"
|
|
]
|
|
},
|
|
"publicSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "public-subnet",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"privateSubnetName": {
|
|
"type": "string",
|
|
"defaultValue": "private-subnet",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"disablePublicIp": {
|
|
"type": "bool",
|
|
"defaultValue": true,
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"adbWorkspaceLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"adbWorkspaceName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"tagValues": {
|
|
"type": "object",
|
|
"defaultValue": {},
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"managedResourceGroupName": "[format('databricks-rg-{0}-{1}', parameters('adbWorkspaceName'), uniqueString(parameters('adbWorkspaceName'), resourceGroup().id))]",
|
|
"managedResourceGroupId": "[format('{0}/resourceGroups/{1}', subscription().id, variables('managedResourceGroupName'))]",
|
|
"vnetId": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Databricks/workspaces",
|
|
"apiVersion": "2018-04-01",
|
|
"name": "[parameters('adbWorkspaceName')]",
|
|
"location": "[parameters('adbWorkspaceLocation')]",
|
|
"sku": {
|
|
"name": "[parameters('adbWorkspaceSkuTier')]"
|
|
},
|
|
"properties": {
|
|
"managedResourceGroupId": "[variables('managedResourceGroupId')]",
|
|
"parameters": {
|
|
"customVirtualNetworkId": {
|
|
"value": "[variables('vnetId')]"
|
|
},
|
|
"customPublicSubnetName": {
|
|
"value": "[parameters('publicSubnetName')]"
|
|
},
|
|
"customPrivateSubnetName": {
|
|
"value": "[parameters('privateSubnetName')]"
|
|
},
|
|
"enableNoPublicIp": {
|
|
"value": "[parameters('disablePublicIp')]"
|
|
}
|
|
}
|
|
},
|
|
"tags": "[parameters('tagValues')]"
|
|
}
|
|
],
|
|
"outputs": {
|
|
"databricks_workspace_id": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))]"
|
|
},
|
|
"databricks_workspaceUrl": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))).workspaceUrl]"
|
|
},
|
|
"databricks_dbfs_storage_accountName": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.Databricks/workspaces', parameters('adbWorkspaceName'))).parameters.storageAccountName.value]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "HubFirewall",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"firewallName": {
|
|
"value": "[variables('firewallName')]"
|
|
},
|
|
"publicIpAddressName": {
|
|
"value": "[variables('firewallPublicIpName')]"
|
|
},
|
|
"vnetName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.hubVnetName.value]"
|
|
},
|
|
"webappDestinationAddresses": {
|
|
"value": "[parameters('webappDestinationAddresses')]"
|
|
},
|
|
"logBlobstorageDomains": {
|
|
"value": "[parameters('logBlobstorageDomains')]"
|
|
},
|
|
"infrastructureDestinationAddresses": {
|
|
"value": "[parameters('extendedInfraIp')]"
|
|
},
|
|
"sccRelayDomains": {
|
|
"value": "[parameters('sccReplayDomain')]"
|
|
},
|
|
"metastoreDomains": {
|
|
"value": "[parameters('metastoreDomains')]"
|
|
},
|
|
"eventHubEndpointDomains": {
|
|
"value": "[parameters('eventHubEndpointDomain')]"
|
|
},
|
|
"artifactBlobStoragePrimaryDomains": {
|
|
"value": "[parameters('artifactBlobStoragePrimaryDomains')]"
|
|
},
|
|
"dbfsBlobStrageDomain": {
|
|
"value": "[array(format('{0}.blob.{1}', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_dbfs_storage_accountName.value, environment().suffixes.storage))]"
|
|
},
|
|
"clientPrivateIpAddr": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ClientPC'), '2019-10-01').outputs.clientPrivateIpaddr.value]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "3290483879050623264"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"firewallLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Azure datacentre Location to deploy the Firewall and IP Address"
|
|
}
|
|
},
|
|
"publicIpAddressName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the IP Address"
|
|
}
|
|
},
|
|
"firewallName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the Azure Firewall"
|
|
}
|
|
},
|
|
"firewallSKU": {
|
|
"type": "string",
|
|
"defaultValue": "Standard",
|
|
"metadata": {
|
|
"description": "Firewall SKU"
|
|
}
|
|
},
|
|
"clientPrivateIpAddr": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Client Private ip address"
|
|
}
|
|
},
|
|
"vnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name of the vnet associated witht he Firewall"
|
|
}
|
|
},
|
|
"webappDestinationAddresses": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of destination IP addresses for Web App"
|
|
}
|
|
},
|
|
"logBlobstorageDomains": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of Log Blob storage domain name"
|
|
}
|
|
},
|
|
"infrastructureDestinationAddresses": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of destination IP addresses for Extended Infrastructure"
|
|
}
|
|
},
|
|
"sccRelayDomains": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of SCC relay domain name"
|
|
}
|
|
},
|
|
"metastoreDomains": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of Metastore domain name"
|
|
}
|
|
},
|
|
"eventHubEndpointDomains": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of Event Hub endpoint domain name"
|
|
}
|
|
},
|
|
"artifactBlobStoragePrimaryDomains": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "List of Artifact Blob storage primary domain name"
|
|
}
|
|
},
|
|
"dbfsBlobStrageDomain": {
|
|
"type": "array",
|
|
"defaultValue": [],
|
|
"metadata": {
|
|
"description": "the domain name of DBFS root Blob storage"
|
|
}
|
|
},
|
|
"fwpublicipcount": {
|
|
"type": "int",
|
|
"defaultValue": 1,
|
|
"metadata": {
|
|
"description": "Specifies the number of public IPs to allocate to the firewall"
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/publicIPAddresses",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[parameters('publicIpAddressName')]",
|
|
"location": "[parameters('firewallLocation')]",
|
|
"sku": {
|
|
"name": "Standard"
|
|
},
|
|
"properties": {
|
|
"publicIPAllocationMethod": "Static"
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/azureFirewalls",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[parameters('firewallName')]",
|
|
"location": "[parameters('firewallLocation')]",
|
|
"properties": {
|
|
"ipConfigurations": [
|
|
{
|
|
"name": "[parameters('publicIpAddressName')]",
|
|
"properties": {
|
|
"subnet": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), 'AzureFirewallSubnet')]"
|
|
},
|
|
"publicIPAddress": {
|
|
"id": "[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))]"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"hubIPAddresses": {
|
|
"publicIPs": {
|
|
"count": "[parameters('fwpublicipcount')]"
|
|
}
|
|
},
|
|
"sku": {
|
|
"tier": "[parameters('firewallSKU')]"
|
|
},
|
|
"threatIntelMode": "Alert",
|
|
"additionalProperties": {
|
|
"Network.DNS.EnableProxy": "true"
|
|
},
|
|
"natRuleCollections": [
|
|
{
|
|
"name": "Allow-RDP-DNAT",
|
|
"properties": {
|
|
"priority": 100,
|
|
"action": {
|
|
"type": "Dnat"
|
|
},
|
|
"rules": [
|
|
{
|
|
"name": "rdp-dnat",
|
|
"protocols": [
|
|
"TCP"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"destinationAddresses": [
|
|
"[reference(resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))).ipAddress]"
|
|
],
|
|
"destinationPorts": [
|
|
"3389"
|
|
],
|
|
"translatedAddress": "[parameters('clientPrivateIpAddr')]",
|
|
"translatedPort": "3389"
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"networkRuleCollections": [
|
|
{
|
|
"name": "Allow-Databricks-Services",
|
|
"properties": {
|
|
"priority": 100,
|
|
"action": {
|
|
"type": "Allow"
|
|
},
|
|
"rules": [
|
|
{
|
|
"name": "Webapp IP",
|
|
"protocols": [
|
|
"TCP"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"destinationAddresses": "[parameters('webappDestinationAddresses')]",
|
|
"sourceIpGroups": [],
|
|
"destinationIpGroups": [],
|
|
"destinationFqdns": [],
|
|
"destinationPorts": [
|
|
"*"
|
|
]
|
|
},
|
|
{
|
|
"name": "Extended infrastructure IP",
|
|
"protocols": [
|
|
"TCP"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"destinationAddresses": "[parameters('infrastructureDestinationAddresses')]",
|
|
"sourceIpGroups": [],
|
|
"destinationIpGroups": [],
|
|
"destinationFqdns": [],
|
|
"destinationPorts": [
|
|
"*"
|
|
]
|
|
},
|
|
{
|
|
"name": "Metastore IP",
|
|
"protocols": [
|
|
"TCP"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"destinationAddresses": [],
|
|
"sourceIpGroups": [],
|
|
"destinationIpGroups": [],
|
|
"destinationFqdns": "[parameters('metastoreDomains')]",
|
|
"destinationPorts": [
|
|
"*"
|
|
]
|
|
},
|
|
{
|
|
"name": "Event Hub endpoint",
|
|
"protocols": [
|
|
"TCP"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"destinationAddresses": [],
|
|
"sourceIpGroups": [],
|
|
"destinationIpGroups": [],
|
|
"destinationFqdns": "[parameters('eventHubEndpointDomains')]",
|
|
"destinationPorts": [
|
|
"*"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"applicationRuleCollections": [
|
|
{
|
|
"name": "Allow-Databricks-Services",
|
|
"properties": {
|
|
"priority": 100,
|
|
"action": {
|
|
"type": "Allow"
|
|
},
|
|
"rules": [
|
|
{
|
|
"name": "Log Blob storage IP",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": "[parameters('logBlobstorageDomains')]",
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "SCC Relay IP",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": "[parameters('sccRelayDomains')]",
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "Artifact Blob storage IP",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": "[parameters('artifactBlobStoragePrimaryDomains')]",
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "DBFS root Blob storage IP",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": "[parameters('dbfsBlobStrageDomain')]",
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"name": "Allow-Websites",
|
|
"properties": {
|
|
"priority": 200,
|
|
"action": {
|
|
"type": "Allow"
|
|
},
|
|
"rules": [
|
|
{
|
|
"name": "Pypi",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": [
|
|
"pypi.org",
|
|
"*.pypi.org",
|
|
"*.pythonhosted.org"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "Maven",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": [
|
|
"*.mvnrepository.com",
|
|
"*.maven.org"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "GitHubScripts",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": [
|
|
"*.githubusercontent.com",
|
|
"github.com"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "LogAnalytics",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": [
|
|
"*.ods.opinsights.azure.com",
|
|
"*.oms.opinsights.azure.com"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
},
|
|
{
|
|
"name": "AzureManagement",
|
|
"protocols": [
|
|
{
|
|
"protocolType": "Http",
|
|
"port": 80
|
|
},
|
|
{
|
|
"protocolType": "Https",
|
|
"port": 443
|
|
}
|
|
],
|
|
"fqdnTags": [],
|
|
"targetFqdns": [
|
|
"[replace(replace(environment().authentication.loginEndpoint, 'https:', ''), '/', '')]",
|
|
"[replace(replace(environment().resourceManager, 'https:', ''), '/', '')]",
|
|
"[format('*.blob.{0}', environment().suffixes.storage)]",
|
|
"*.azure-automation.net",
|
|
"ml.azure.com",
|
|
"*.msauth.net"
|
|
],
|
|
"sourceAddresses": [
|
|
"*"
|
|
],
|
|
"sourceIpGroups": []
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/publicIPAddresses', parameters('publicIpAddressName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"firewallPrivateIp": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.Network/azureFirewalls', parameters('firewallName'))).ipConfigurations[0].properties.privateIPAddress]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ClientPC')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "RouteTableUpdate",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"routeTableName": {
|
|
"value": "[variables('fwRoutingTable')]"
|
|
},
|
|
"firewallPrivateIp": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubFirewall'), '2019-10-01').outputs.firewallPrivateIp.value]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "1265305905541722464"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"routeTableName": {
|
|
"type": "string"
|
|
},
|
|
"firewallPrivateIp": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/routeTables/routes",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[format('{0}/Firewall-Route', parameters('routeTableName'))]",
|
|
"properties": {
|
|
"addressPrefix": "0.0.0.0/0",
|
|
"nextHopType": "VirtualAppliance",
|
|
"nextHopIpAddress": "[parameters('firewallPrivateIp')]",
|
|
"hasBgpOverride": false
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubFirewall')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "KeyVault",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"keyVaultName": {
|
|
"value": "[variables('keyVaultName')]"
|
|
},
|
|
"objectId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ManagedIdentity'), '2019-10-01').outputs.mIdentityClientId.value]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "1226844121658064294"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"keyVaultName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Specifies the name of the key vault."
|
|
}
|
|
},
|
|
"keyVaultLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Specifies the Azure location where the key vault should be created."
|
|
}
|
|
},
|
|
"enabledForDeployment": {
|
|
"type": "bool",
|
|
"defaultValue": false,
|
|
"metadata": {
|
|
"description": "Specifies whether Azure Virtual Machines are permitted to retrieve certificates stored as secrets from the key vault."
|
|
},
|
|
"allowedValues": [
|
|
true,
|
|
false
|
|
]
|
|
},
|
|
"enabledForTemplateDeployment": {
|
|
"type": "bool",
|
|
"defaultValue": false,
|
|
"metadata": {
|
|
"description": "Specifies whether Azure Resource Manager is permitted to retrieve secrets from the key vault."
|
|
},
|
|
"allowedValues": [
|
|
true,
|
|
false
|
|
]
|
|
},
|
|
"tenantId": {
|
|
"type": "string",
|
|
"defaultValue": "[subscription().tenantId]",
|
|
"metadata": {
|
|
"description": "Specifies the Azure Active Directory tenant ID that should be used for authenticating requests to the key vault. Get it by using Get-AzSubscription cmdlet."
|
|
}
|
|
},
|
|
"objectId": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Specifies the object ID of a user, service principal or security group in the Azure Active Directory tenant for the vault. The object ID must be unique for the list of access policies. Get it by using Get-AzADUser or Get-AzADServicePrincipal cmdlets."
|
|
}
|
|
},
|
|
"keyVaultSkuTier": {
|
|
"type": "string",
|
|
"defaultValue": "standard",
|
|
"metadata": {
|
|
"description": "Specifies whether the key vault is a standard vault or a premium vault."
|
|
},
|
|
"allowedValues": [
|
|
"standard",
|
|
"premium"
|
|
]
|
|
},
|
|
"tagValues": {
|
|
"type": "object",
|
|
"defaultValue": {}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.KeyVault/vaults",
|
|
"apiVersion": "2019-09-01",
|
|
"name": "[parameters('keyVaultName')]",
|
|
"location": "[parameters('keyVaultLocation')]",
|
|
"properties": {
|
|
"enabledForDeployment": "[parameters('enabledForDeployment')]",
|
|
"enabledForTemplateDeployment": "[parameters('enabledForTemplateDeployment')]",
|
|
"tenantId": "[parameters('tenantId')]",
|
|
"accessPolicies": [
|
|
{
|
|
"objectId": "[parameters('objectId')]",
|
|
"tenantId": "[parameters('tenantId')]",
|
|
"permissions": {
|
|
"secrets": [
|
|
"list",
|
|
"get",
|
|
"set"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"sku": {
|
|
"name": "[parameters('keyVaultSkuTier')]",
|
|
"family": "A"
|
|
},
|
|
"softDeleteRetentionInDays": 7,
|
|
"networkAcls": {
|
|
"defaultAction": "Deny",
|
|
"bypass": "AzureServices",
|
|
"ipRules": [],
|
|
"virtualNetworkRules": []
|
|
}
|
|
},
|
|
"tags": "[parameters('tagValues')]"
|
|
}
|
|
],
|
|
"outputs": {
|
|
"keyvault_id": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))]"
|
|
},
|
|
"keyvault_uri": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName'))).vaultUri]"
|
|
},
|
|
"keyvaultResource": {
|
|
"type": "object",
|
|
"value": "[reference(resourceId('Microsoft.KeyVault/vaults', parameters('keyVaultName')), '2019-09-01', 'full')]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ManagedIdentity')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "ClientPC",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"adminUsername": {
|
|
"value": "[parameters('adminUsername')]"
|
|
},
|
|
"adminPassword": {
|
|
"value": "[parameters('adminPassword')]"
|
|
},
|
|
"vnetName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.hubVnetName.value]"
|
|
},
|
|
"clientPcName": {
|
|
"value": "[variables('clientPcName')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "15240521664149869021"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"clientPcName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"osDiskType": {
|
|
"type": "string",
|
|
"defaultValue": "Premium_LRS",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"virtualMachineSize": {
|
|
"type": "string",
|
|
"defaultValue": "Standard_DS1_v2",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"patchMode": {
|
|
"type": "string",
|
|
"defaultValue": "AutomaticByOS",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"vnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"adminUsername": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"location": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
},
|
|
"adminPassword": {
|
|
"type": "secureString",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"networkInterfaceName": "[format('{0}-Iface', parameters('clientPcName'))]",
|
|
"virtualMachineName": "[parameters('clientPcName')]",
|
|
"subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('vnetName'), 'ClientDevices')]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/networkInterfaces",
|
|
"apiVersion": "2018-10-01",
|
|
"name": "[variables('networkInterfaceName')]",
|
|
"location": "[parameters('location')]",
|
|
"properties": {
|
|
"ipConfigurations": [
|
|
{
|
|
"name": "ipconfig1",
|
|
"properties": {
|
|
"subnet": {
|
|
"id": "[variables('subnetRef')]"
|
|
},
|
|
"privateIPAllocationMethod": "Dynamic"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Compute/virtualMachines",
|
|
"apiVersion": "2021-03-01",
|
|
"name": "[variables('virtualMachineName')]",
|
|
"location": "[parameters('location')]",
|
|
"properties": {
|
|
"hardwareProfile": {
|
|
"vmSize": "[parameters('virtualMachineSize')]"
|
|
},
|
|
"storageProfile": {
|
|
"osDisk": {
|
|
"createOption": "FromImage",
|
|
"managedDisk": {
|
|
"storageAccountType": "[parameters('osDiskType')]"
|
|
}
|
|
},
|
|
"imageReference": {
|
|
"publisher": "MicrosoftWindowsDesktop",
|
|
"offer": "Windows-10",
|
|
"sku": "20h2-pro-g2",
|
|
"version": "latest"
|
|
}
|
|
},
|
|
"networkProfile": {
|
|
"networkInterfaces": [
|
|
{
|
|
"id": "[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]"
|
|
}
|
|
]
|
|
},
|
|
"osProfile": {
|
|
"computerName": "[variables('virtualMachineName')]",
|
|
"adminUsername": "[parameters('adminUsername')]",
|
|
"adminPassword": "[parameters('adminPassword')]",
|
|
"windowsConfiguration": {
|
|
"enableAutomaticUpdates": true,
|
|
"provisionVMAgent": true,
|
|
"patchSettings": {
|
|
"enableHotpatching": false,
|
|
"patchMode": "[parameters('patchMode')]"
|
|
}
|
|
}
|
|
},
|
|
"licenseType": "Windows_Client",
|
|
"diagnosticsProfile": {
|
|
"bootDiagnostics": {
|
|
"enabled": true
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"clientPrivateIpaddr": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.Network/networkInterfaces', variables('networkInterfaceName'))).ipConfigurations[0].properties.privateIPAddress]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "LogAnalytics",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "8904458932555451638"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"logAnalyticsWkspName": {
|
|
"type": "string",
|
|
"defaultValue": "[toLower(format('spark-monitoring-{0}', uniqueString(resourceGroup().name)))]"
|
|
},
|
|
"logAnalyticsWkspLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]"
|
|
},
|
|
"logAnalyticsWkspSku": {
|
|
"type": "string",
|
|
"defaultValue": "Standalone",
|
|
"metadata": {
|
|
"description": "Service Tier: Free, Standalone, PerNode, or PerGB2018"
|
|
},
|
|
"allowedValues": [
|
|
"Free",
|
|
"Standalone",
|
|
"PerNode",
|
|
"PerGB2018"
|
|
]
|
|
},
|
|
"logAnalyticsWkspRentationDays": {
|
|
"type": "int",
|
|
"defaultValue": 30,
|
|
"metadata": {
|
|
"description": "Number of days of retention. Free plans can only have 7 days, Standalone and Log Analytics plans include 30 days for free"
|
|
},
|
|
"maxValue": 730,
|
|
"minValue": 7
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"queries": [
|
|
{
|
|
"displayName": "Stage Latency Per Stage (Stage Duration)",
|
|
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n | extend stageDuration=Stage_Info_Completion_Time_d - Stage_Info_Submission_Time_d\n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s) \n| extend stageDuration=Stage_Info_Completion_Time_d - Stage_Info_Submission_Time_d \n| summarize percentiles(stageDuration,10,30,50,90) by bin(TimeGenerated, 1m), slice\n| order by TimeGenerated asc nulls last\n\n"
|
|
},
|
|
{
|
|
"displayName": "Stage Throughput Per Stage",
|
|
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project \nStage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(\"# StagesCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",\napptag,\" \",Stage_Info_Stage_Name_s) \n| summarize StagesCompleted=count(Event_s) by bin(TimeGenerated,1m), slice\n| order by TimeGenerated asc nulls last\n\n"
|
|
},
|
|
{
|
|
"displayName": "Tasks Per Stage",
|
|
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project \nStage_Info_Stage_ID_d,apptag,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerStageCompleted\" \n) on Stage_Info_Stage_ID_d;\nresults\n | extend slice = strcat(\"# StagesCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s) \n| project Stage_Info_Number_of_Tasks_d,slice,TimeGenerated \n| order by TimeGenerated asc nulls last\n\n"
|
|
},
|
|
{
|
|
"displayName": "% Serialize Time Per Executor",
|
|
"query": "let results = SparkMetric_CL\n| where name_s contains \"executor.resultserializationtime\" \n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , setime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend serUsage=(setime/runTime)*100\n| summarize SerializationCpuTime=percentile(serUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n| render timechart "
|
|
},
|
|
{
|
|
"displayName": "Shuffle Bytes Read Per Executor",
|
|
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.shuffleTotalBytesRead\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.shuffleTotalBytesRead\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize max(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
|
|
},
|
|
{
|
|
"displayName": "Error Traces (Bad Record Or Bad Files)",
|
|
"query": "SparkListenerEvent_CL\r\n| where Level contains \"Error\"\r\n| project TimeGenerated , Message \r\n"
|
|
},
|
|
{
|
|
"displayName": "Task Shuffle Bytes Written",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend schedulerdelay = Task_Info_Launch_Time_d - Stage_Info_Submission_Time_d\n| extend name=strcat(\"SchuffleBytesWritten \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Task Input Bytes Read",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,Task_Metrics_Input_Metrics_Bytes_Read_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"InputBytesRead \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Input_Metrics_Bytes_Read_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Sum Task Execution Per Host",
|
|
"query": "SparkListenerEvent_CL\n| where Event_s contains \"taskend\" \n| extend taskDuration=Task_Info_Finish_Time_d-Task_Info_Launch_Time_d \n| summarize sum(taskDuration) by bin(TimeGenerated, 1m), Task_Info_Host_s\n| order by TimeGenerated asc nulls last "
|
|
},
|
|
{
|
|
"displayName": "% CPU Time Per Executor",
|
|
"query": "let results = SparkMetric_CL \n| where name_s contains \"executor.cpuTime\" \n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , cpuTime=count_d/1000000 , executor ,name_s\n| join kind= inner (\n SparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend cpuUsage=(cpuTime/runTime)*100\n| summarize ExecutorCpuTime = percentile(cpuUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last \n"
|
|
},
|
|
{
|
|
"displayName": "Job Throughput",
|
|
"query": "let results=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerJobStart\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\n| project Job_ID_d,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerJobEnd\"\n | where Job_Result_Result_s contains \"JobSucceeded\"\n | project Event_s,Job_ID_d,TimeGenerated\n) on Job_ID_d;\nresults\n| extend slice=strcat(\"#JobsCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag)\n| summarize count(Event_s) by bin(TimeGenerated, 1m),slice\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Disk Bytes Spilled Per Executor",
|
|
"query": "let results=SparkMetric_CL\r\n| where name_s contains \"executor.diskBytesSpilled\"\r\n| extend sname=split(name_s, \".\") \r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkMetric_CL\r\n | where name_s contains \"executor.diskBytesSpilled\"\r\n | extend sname=split(name_s, \".\") \r\n | extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\r\n) on executor, TimeGenerated;\r\nresults\r\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \r\n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last\r\n"
|
|
},
|
|
{
|
|
"displayName": "Task Shuffle Read Time",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskShuffleReadTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Heap Memory Per Executor",
|
|
"query": "SparkMetric_CL\n| where name_s contains \"shuffle-client.usedHeapMemory\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Job Errors Per Job",
|
|
"query": "let results=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerJobStart\"\r\n| project Job_ID_d,Properties_callSite_short_s,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerJobEnd\"\r\n | where Job_Result_Result_s !contains \"JobSucceeded\"\r\n | project Event_s,Job_ID_d,TimeGenerated\r\n) on Job_ID_d;\r\nresults\r\n| extend slice=strcat(\"JobErrors \",Properties_callSite_short_s)\r\n| summarize count(Event_s) by bin(TimeGenerated, 1m),slice\r\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Task Errors Per Stage",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageCompleted\"\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Event_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s !contains \"Success\"\n | project Stage_ID_d,Task_Info_Task_ID_d,Task_End_Reason_Reason_s,\n TaskEvent=Event_s,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend slice=strcat(\"#TaskErrors \",Stage_Info_Stage_Name_s)\n| summarize count(TaskEvent) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last\n"
|
|
},
|
|
{
|
|
"displayName": "Streaming Latency Per Stream",
|
|
"query": "\r\n\r\nSparkListenerEvent_CL\r\n| where Event_s contains \"queryprogressevent\"\r\n| extend sname=strcat(progress_name_s,\"-\",\"triggerexecution\") \r\n| summarize percentile(progress_durationMs_triggerExecution_d,90) by bin(TimeGenerated, 1m), sname\r\n| order by TimeGenerated asc nulls last \r\n"
|
|
},
|
|
{
|
|
"displayName": "Task Shuffle Write Time",
|
|
"query": "let result=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerStageCompleted\"\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerTaskEnd\"\r\n | where Task_End_Reason_Reason_s contains \"Success\"\r\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\r\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\r\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\r\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\r\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\r\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\r\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\r\nresult\r\n| extend ShuffleWriteTime=Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d/1000000\r\n| extend name=strcat(\"TaskShuffleWriteTime \",Stage_Info_Stage_Name_s)\r\n| summarize percentile(ShuffleWriteTime,90) by bin(TimeGenerated,1m),name\r\n| order by TimeGenerated asc nulls last;\r\n\r\n"
|
|
},
|
|
{
|
|
"displayName": "Task Deserialization Time",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,Task_Metrics_Input_Metrics_Bytes_Read_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskDeserializationTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Executor_Deserialize_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Task Result Serialization Time",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"TaskResultSerializationTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Result_Serialization_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "File System Bytes Read Per Executor",
|
|
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| where name_s contains \"executor.filesystem.file.read_bytes\" \n| summarize FileSystemReadBytes=percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Streaming Throughput Processed Rows/Sec",
|
|
"query": "SparkListenerEvent_CL\r\n| where Event_s contains \"progress\"\r\n| extend sname=strcat(progress_name_s,\"-ProcRowsPerSecond\") \r\n| extend status = todouble(extractjson(\"$.[0].processedRowsPerSecond\", progress_sources_s))\r\n| summarize percentile(status,90) by bin(TimeGenerated, 1m) , sname\r\n| order by TimeGenerated asc nulls last "
|
|
},
|
|
{
|
|
"displayName": "% Deserialize Time Per Executor",
|
|
"query": "let results = SparkMetric_CL \n| where name_s contains \"executor.deserializetime\" \n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , desetime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend deseUsage=(desetime/runTime)*100\n| summarize deSerializationCpuTime=percentiles(deseUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last "
|
|
},
|
|
{
|
|
"displayName": "Tasks Per Executor (Sum Of Tasks Per Executor)",
|
|
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1]) \n| where name_s contains \"threadpool.activeTasks\" \n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m),executor\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "File System Bytes Write Per Executor",
|
|
"query": "SparkMetric_CL\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| where name_s contains \"executor.filesystem.file.write_bytes\" \n| summarize FileSystemWriteBytes=percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last "
|
|
},
|
|
{
|
|
"displayName": "Task Scheduler Delay Latency",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend schedulerdelay = Task_Info_Launch_Time_d - Stage_Info_Submission_Time_d\n| extend name=strcat(\"SchedulerDelayTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(schedulerdelay,90) , percentile(Task_Metrics_Executor_Run_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Streaming Errors Per Stream",
|
|
"query": "SparkListenerEvent_CL\r\n| extend slice = strcat(\"CountExceptions\",progress_name_s) \r\n| where Level contains \"Error\"\r\n| summarize count(Level) by bin(TimeGenerated, 1m), slice \r\n"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Client Memory Per Executor",
|
|
"query": "SparkMetric_CL\r\n| where name_s contains \"shuffle-client.usedDirectMemory\"\r\n| extend sname=split(name_s, \".\")\r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Job Latency Per Job (Batch Duration)",
|
|
"query": "let results=SparkListenerEvent_CL\r\n| where Event_s contains \"SparkListenerJobStart\"\r\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Job_ID_d,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,\r\nSubmission_Time_d,TimeGenerated\r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkListenerEvent_CL\r\n | where Event_s contains \"SparkListenerJobEnd\"\r\n | where Job_Result_Result_s contains \"JobSucceeded\"\r\n | project Event_s,Job_ID_d,Completion_Time_d,TimeGenerated\r\n) on Job_ID_d;\r\nresults\r\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag)\r\n| extend jobDuration=Completion_Time_d - Submission_Time_d \r\n| summarize percentiles(jobDuration,10,30,50,90) by bin(TimeGenerated, 1m), slice\r\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Task Executor Compute Time (Data Skew Time)",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last\n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"ExecutorComputeTime \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Executor_Run_Time_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Streaming Throughput Input Rows/Sec",
|
|
"query": "SparkListenerEvent_CL\r\n| where Event_s contains \"progress\"\r\n| extend sname=strcat(progress_name_s,\"-inputRowsPerSecond\") \r\n| extend status = todouble(extractjson(\"$.[0].inputRowsPerSecond\", progress_sources_s))\r\n| summarize percentile(status,90) by bin(TimeGenerated, 1m) , sname\r\n| order by TimeGenerated asc nulls last \n"
|
|
},
|
|
{
|
|
"displayName": "Task Shuffle Bytes Read",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last\n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Metrics_Executor_Deserialize_Time_d,Task_Metrics_Shuffle_Read_Metrics_Fetch_Wait_Time_d,\n Task_Metrics_Executor_Run_Time_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Write_Time_d,\n Task_Metrics_Result_Serialization_Time_d,Task_Info_Getting_Result_Time_d,\n Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,Task_Metrics_Shuffle_Write_Metrics_Shuffle_Bytes_Written_d,\n Task_Metrics_JVM_GC_Time_d,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend name=strcat(\"SchuffleBytesRead \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize percentile(Task_Metrics_Shuffle_Read_Metrics_Remote_Bytes_Read_d,90) by bin(TimeGenerated,1m),name\n| order by TimeGenerated asc nulls last;\n\n"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Memory Bytes Spilled Per Executor",
|
|
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.memoryBytesSpilled\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.memoryBytesSpilled\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
|
|
},
|
|
{
|
|
"displayName": "% JVM Time Per Executor",
|
|
"query": "let results = SparkMetric_CL\n| where name_s contains \"executor.jvmGCTime\" \n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , jvmgcTime=count_d , executor ,name_s\n| join kind= inner (\nSparkMetric_CL\n| where name_s contains \"executor.RunTime\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| project TimeGenerated , runTime=count_d , executor ,name_s\n) on executor, TimeGenerated;\nresults\n| extend JvmcpuUsage=(jvmgcTime/runTime)*100\n| summarize JvmCpuTime = percentile(JvmcpuUsage,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n| render timechart \n"
|
|
},
|
|
{
|
|
"displayName": "Running Executors",
|
|
"query": "SparkMetric_CL\n| where name_s !contains \"driver\" \n| where name_s contains \"executor\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[1]) \n| extend app=strcat(sname[0])\n| summarize NumExecutors=dcount(executor) by bin(TimeGenerated, 1m),app\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Bytes Read To Disk Per Executor",
|
|
"query": "let results=SparkMetric_CL\r\n| where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\r\n| extend sname=split(name_s, \".\") \r\n| extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \r\n| order by TimeGenerated asc nulls last \r\n| join kind= inner (\r\n SparkMetric_CL\r\n | where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\r\n | extend sname=split(name_s, \".\") \r\n | extend executor=strcat(\"executorid:\",sname[1])\r\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\r\n) on executor, TimeGenerated;\r\nresults\r\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \r\n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\r\n| order by TimeGenerated asc nulls last\r\n"
|
|
},
|
|
{
|
|
"displayName": "Task Latency Per Stage (Tasks Duration)",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,apptag,Properties_spark_databricks_clusterUsageTags_clusterName_s,Event_s,TimeGenerated\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Task_Info_Launch_Time_d,Stage_ID_d,Task_Info_Task_ID_d,Event_s,\n Task_Info_Finish_Time_d\n ) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend TaskLatency = Task_Info_Finish_Time_d - Task_Info_Launch_Time_d\n| extend slice=strcat(Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\"-\",Stage_Info_Stage_Name_s)\n| summarize percentile(TaskLatency,90) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last;\n"
|
|
},
|
|
{
|
|
"displayName": "Task Throughput (Sum Of Tasks Per Stage)",
|
|
"query": "let result=SparkListenerEvent_CL\n| where Event_s contains \"SparkListenerStageSubmitted\"\n| extend metricsns=columnifexists(\"Properties_spark_metrics_namespace_s\",Properties_spark_app_id_s)\r\n| extend apptag=iif(isnotempty(metricsns),metricsns,Properties_spark_app_id_s)\r\n| project Stage_Info_Stage_ID_d,Stage_Info_Stage_Name_s,Stage_Info_Submission_Time_d,Event_s,TimeGenerated,Properties_spark_databricks_clusterUsageTags_clusterName_s,apptag\n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkListenerEvent_CL\n | where Event_s contains \"SparkListenerTaskEnd\"\n | where Task_End_Reason_Reason_s contains \"Success\"\n | project Stage_ID_d,Task_Info_Task_ID_d,\n TaskEvent=Event_s,TimeGenerated\n) on $left.Stage_Info_Stage_ID_d == $right.Stage_ID_d;\nresult\n| extend slice=strcat(\"#TasksCompleted \",Properties_spark_databricks_clusterUsageTags_clusterName_s,\"-\",apptag,\" \",Stage_Info_Stage_Name_s)\n| summarize count(TaskEvent) by bin(TimeGenerated,1m),slice\n| order by TimeGenerated asc nulls last\n"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Client Direct Memory",
|
|
"query": "SparkMetric_CL\n| where name_s contains \"shuffle-client.usedDirectMemory\"\n| extend sname=split(name_s, \".\")\n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize percentile(value_d,90) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last"
|
|
},
|
|
{
|
|
"displayName": "Disk Bytes Spilled",
|
|
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.diskBytesSpilled\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.diskBytesSpilled\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
|
|
},
|
|
{
|
|
"displayName": "Shuffle Bytes Read",
|
|
"query": "let results=SparkMetric_CL\n| where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\n| extend sname=split(name_s, \".\") \n| extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MaxShuffleWrites=max(count_d) by bin(TimeGenerated, 1m), executor \n| order by TimeGenerated asc nulls last \n| join kind= inner (\n SparkMetric_CL\n | where name_s contains \"executor.shuffleRemoteBytesReadToDisk\"\n | extend sname=split(name_s, \".\") \n | extend executor=strcat(sname[0],\".\",sname[1])\n| summarize MinShuffleWrites=min(count_d) by bin(TimeGenerated, 1m), executor\n) on executor, TimeGenerated;\nresults\n| extend ShuffleBytesWritten=MaxShuffleWrites-MinShuffleWrites \n| summarize any(ShuffleBytesWritten) by bin(TimeGenerated, 1m), executor\n| order by TimeGenerated asc nulls last\n"
|
|
}
|
|
]
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.OperationalInsights/workspaces",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "[parameters('logAnalyticsWkspName')]",
|
|
"location": "[parameters('logAnalyticsWkspLocation')]",
|
|
"properties": {
|
|
"sku": {
|
|
"name": "[parameters('logAnalyticsWkspSku')]"
|
|
},
|
|
"retentionInDays": "[parameters('logAnalyticsWkspRentationDays')]",
|
|
"features": {
|
|
"enableDataExport": true
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"copy": {
|
|
"name": "WkspSearch",
|
|
"count": "[length(variables('queries'))]"
|
|
},
|
|
"type": "Microsoft.OperationalInsights/workspaces/savedSearches",
|
|
"apiVersion": "2020-08-01",
|
|
"name": "[format('{0}/{1}', parameters('logAnalyticsWkspName'), guid(format('{0}{1}{2}', resourceGroup().id, deployment().name, copyIndex())))]",
|
|
"properties": {
|
|
"category": "Spark Metrics",
|
|
"displayName": "[variables('queries')[copyIndex()].displayName]",
|
|
"query": "[variables('queries')[copyIndex()].query]",
|
|
"etag": "*"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"logAnalyticsWkspId": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName'))).customerId]"
|
|
},
|
|
"primarySharedKey": {
|
|
"type": "string",
|
|
"value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName')), '2020-10-01').primarySharedKey]"
|
|
},
|
|
"secondarySharedKey": {
|
|
"type": "string",
|
|
"value": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', parameters('logAnalyticsWkspName')), '2020-10-01').secondarySharedKey]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "EventHub",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"namespaceName": {
|
|
"value": "[variables('eHNameSpace')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "11175130669690033926"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"namespaceName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Name for the Event Hub cluster."
|
|
}
|
|
},
|
|
"eventHubName": {
|
|
"type": "string",
|
|
"defaultValue": "[parameters('namespaceName')]",
|
|
"metadata": {
|
|
"description": "Name for the Event Hub to be created in the Event Hub namespace within the Event Hub cluster."
|
|
}
|
|
},
|
|
"location": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Specifies the Azure location for all resources."
|
|
}
|
|
},
|
|
"eHRuleName": {
|
|
"type": "string",
|
|
"defaultValue": "rule",
|
|
"metadata": {
|
|
"description": ""
|
|
}
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.EventHub/namespaces",
|
|
"apiVersion": "2021-01-01-preview",
|
|
"name": "[parameters('namespaceName')]",
|
|
"location": "[parameters('location')]",
|
|
"sku": {
|
|
"name": "Standard",
|
|
"tier": "Standard",
|
|
"capacity": 1
|
|
},
|
|
"properties": {
|
|
"isAutoInflateEnabled": false,
|
|
"maximumThroughputUnits": 0
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.EventHub/namespaces/eventhubs",
|
|
"apiVersion": "2021-01-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('namespaceName'), parameters('eventHubName'))]",
|
|
"properties": {
|
|
"messageRetentionInDays": 7,
|
|
"partitionCount": 1
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.EventHub/namespaces', parameters('namespaceName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.EventHub/namespaces/eventhubs/authorizationRules",
|
|
"apiVersion": "2021-01-01-preview",
|
|
"name": "[format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName'))]",
|
|
"properties": {
|
|
"rights": [
|
|
"Send",
|
|
"Listen"
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaceName'), parameters('eventHubName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"eHNamespaceId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.EventHub/namespaces', parameters('namespaceName'))]"
|
|
},
|
|
"eHubNameId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.EventHub/namespaces/eventhubs', parameters('namespaceName'), parameters('eventHubName'))]"
|
|
},
|
|
"eHAuthRulesId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[0], split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[1], split(format('{0}/{1}/{2}', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '/')[2])]"
|
|
},
|
|
"eHPConnString": {
|
|
"type": "string",
|
|
"value": "[listKeys(resourceId('Microsoft.EventHub/namespaces/eventhubs/authorizationRules', parameters('namespaceName'), parameters('eventHubName'), parameters('eHRuleName')), '2021-01-01-preview').primaryConnectionString]"
|
|
},
|
|
"eHName": {
|
|
"type": "string",
|
|
"value": "[parameters('eventHubName')]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "PrivateEndPoints",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"keyvaultPrivateLinkResource": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_id.value]"
|
|
},
|
|
"privateLinkSubnetId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.privatelinksubnet_id.value]"
|
|
},
|
|
"storageAccountName": {
|
|
"value": "StorageAccount"
|
|
},
|
|
"storageAccountPrivateLinkResource": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.storageaccount_id.value]"
|
|
},
|
|
"eventHubName": {
|
|
"value": "[variables('eventHubName')]"
|
|
},
|
|
"eventHubPrivateLinkResource": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHNamespaceId.value]"
|
|
},
|
|
"AmlName": {
|
|
"value": "MLWorkspace"
|
|
},
|
|
"amlPrivateLinkResource": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'MLWorkspace'), '2019-10-01').outputs.amlId.value]"
|
|
},
|
|
"vnetName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.spokeVnetName.value]"
|
|
},
|
|
"containerRegistryName": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'MLWorkspace'), '2019-10-01').outputs.ctrRegistryName.value]"
|
|
},
|
|
"crPrivateLinkResource": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'MLWorkspace'), '2019-10-01').outputs.ctrRegistryId.value]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "2805453972849292259"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"storageAccountPrivateLinkResource": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Storage Account Privatelink Resource"
|
|
}
|
|
},
|
|
"storageAccountName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Storage Account name"
|
|
}
|
|
},
|
|
"keyvaultPrivateLinkResource": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Keyvault Private Link resource."
|
|
}
|
|
},
|
|
"keyvaultName": {
|
|
"type": "string",
|
|
"defaultValue": "KeyVault",
|
|
"metadata": {
|
|
"description": "keyvault name."
|
|
}
|
|
},
|
|
"eventHubName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "event hub name."
|
|
}
|
|
},
|
|
"eventHubPrivateLinkResource": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "EventHub Private Link resource."
|
|
}
|
|
},
|
|
"vnetName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Vnet name for private link"
|
|
}
|
|
},
|
|
"privateLinkSubnetId": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "Privatelink subnet Id"
|
|
}
|
|
},
|
|
"privateLinkLocation": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]",
|
|
"metadata": {
|
|
"description": "Privatelink subnet Id"
|
|
}
|
|
},
|
|
"AmlName": {
|
|
"type": "string"
|
|
},
|
|
"amlPrivateLinkResource": {
|
|
"type": "string"
|
|
},
|
|
"containerRegistryName": {
|
|
"type": "string"
|
|
},
|
|
"crPrivateLinkResource": {
|
|
"type": "string"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"privateDnsNameStorageDfs_var": "[format('privatelink.dfs.{0}', environment().suffixes.storage)]",
|
|
"privateDnsNameStorageBlob_var": "[format('privatelink.blob.{0}', environment().suffixes.storage)]",
|
|
"privateDnsNameStorageFile_var": "[format('privatelink.file.{0}', environment().suffixes.storage)]",
|
|
"adlsPrivateDnsZoneName_var": "[format('{0}-dfs-Privateendpoint', toLower(parameters('storageAccountName')))]",
|
|
"filePrivateDnsZoneName_var": "[format('{0}-file-Privateendpoint', toLower(parameters('storageAccountName')))]",
|
|
"blobPrivateDnsZoneName_var": "[format('{0}-blob-Privateendpoint', toLower(parameters('storageAccountName')))]",
|
|
"privateDnsNameVault_var": "privatelink.vaultcore.azure.net",
|
|
"keyvaultPrivateEndpointName_var": "[format('{0}-Privateendpoint', toLower(parameters('keyvaultName')))]",
|
|
"privateDnsNameEventHub_var": "privatelink.servicebus.windows.net",
|
|
"eventHubPrivateEndpointName_var": "[format('{0}-Privateendpoint', parameters('eventHubName'))]",
|
|
"privateDnsNameAmlApi_var": "privatelink.api.azureml.ms",
|
|
"privateDnsNameAmlNotebook_var": "privatelink.notebooks.azure.net",
|
|
"amlPrivateEndpointName_var": "[format('{0}-Privateendpoint', parameters('AmlName'))]",
|
|
"privateDnsNameCr_var": "privatelink.azurecr.io",
|
|
"crPrivateEndpointName_var": "[format('{0}-Privateendpoint', parameters('containerRegistryName'))]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('adlsPrivateDnsZoneName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('adlsPrivateDnsZoneName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('storageAccountPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"dfs"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('filePrivateDnsZoneName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('filePrivateDnsZoneName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('storageAccountPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"file"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('blobPrivateDnsZoneName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('blobPrivateDnsZoneName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('storageAccountPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"blob"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameStorageDfs_var')]",
|
|
"location": "global",
|
|
"tags": {},
|
|
"properties": {},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('adlsPrivateDnsZoneName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameStorageFile_var')]",
|
|
"location": "global",
|
|
"tags": {},
|
|
"properties": {},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('filePrivateDnsZoneName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameStorageBlob_var')]",
|
|
"location": "global",
|
|
"tags": {},
|
|
"properties": {},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('blobPrivateDnsZoneName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameStorageDfs_var'), format('dfs_link_to_{0}', toLower(parameters('vnetName'))))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageDfs_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameStorageFile_var'), format('file_link_to_{0}', toLower(parameters('vnetName'))))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageFile_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameStorageBlob_var'), format('file_link_to_{0}', toLower(parameters('vnetName'))))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageBlob_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('adlsPrivateDnsZoneName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-dfs-core-windows-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageDfs_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageDfs_var'))]",
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('adlsPrivateDnsZoneName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('filePrivateDnsZoneName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-file-core-windows-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageFile_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('filePrivateDnsZoneName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageFile_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('blobPrivateDnsZoneName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-blob-core-windows-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageBlob_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('blobPrivateDnsZoneName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameStorageBlob_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('keyvaultPrivateEndpointName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('keyvaultPrivateEndpointName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('keyvaultPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"vault"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
},
|
|
"tags": {}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameVault_var')]",
|
|
"location": "global",
|
|
"tags": {},
|
|
"properties": {},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('keyvaultPrivateEndpointName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameVault_var'), parameters('vnetName'))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('keyvaultPrivateEndpointName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-vaultcore-azure-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('keyvaultPrivateEndpointName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameVault_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('eventHubPrivateEndpointName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('eventHubPrivateEndpointName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('eventHubPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"namespace"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameEventHub_var')]",
|
|
"location": "global",
|
|
"tags": {},
|
|
"properties": {},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('eventHubPrivateEndpointName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameEventHub_var'), parameters('vnetName'))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('eventHubPrivateEndpointName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-servicebus-windows-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('eventHubPrivateEndpointName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameEventHub_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('amlPrivateEndpointName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('amlPrivateEndpointName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('amlPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"amlworkspace"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2018-09-01",
|
|
"name": "[variables('privateDnsNameAmlApi_var')]",
|
|
"location": "global",
|
|
"properties": {}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2018-09-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameAmlApi_var'), 'privatelink_api_azureml_ms')]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlApi_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2018-09-01",
|
|
"name": "[variables('privateDnsNameAmlNotebook_var')]",
|
|
"location": "global",
|
|
"properties": {}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2018-09-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameAmlNotebook_var'), 'privatelink_notebooks_azure_net')]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlNotebook_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2020-03-01",
|
|
"name": "[format('{0}/{1}', variables('amlPrivateEndpointName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-api-azureml-ms",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlApi_var'))]"
|
|
}
|
|
},
|
|
{
|
|
"name": "privatelink-notebooks-azure-net",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlNotebook_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('amlPrivateEndpointName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlApi_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameAmlNotebook_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[variables('crPrivateEndpointName_var')]",
|
|
"location": "[parameters('privateLinkLocation')]",
|
|
"properties": {
|
|
"privateLinkServiceConnections": [
|
|
{
|
|
"name": "[variables('crPrivateEndpointName_var')]",
|
|
"properties": {
|
|
"privateLinkServiceId": "[parameters('crPrivateLinkResource')]",
|
|
"groupIds": [
|
|
"registry"
|
|
]
|
|
}
|
|
}
|
|
],
|
|
"subnet": {
|
|
"id": "[parameters('privateLinkSubnetId')]"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[variables('privateDnsNameCr_var')]",
|
|
"location": "global",
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('crPrivateEndpointName_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateDnsZones/virtualNetworkLinks",
|
|
"apiVersion": "2020-06-01",
|
|
"name": "[format('{0}/{1}', variables('privateDnsNameCr_var'), parameters('vnetName'))]",
|
|
"location": "global",
|
|
"properties": {
|
|
"virtualNetwork": {
|
|
"id": "[resourceId('Microsoft.Network/virtualNetworks', parameters('vnetName'))]"
|
|
},
|
|
"registrationEnabled": false
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameCr_var'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Network/privateEndpoints/privateDnsZoneGroups",
|
|
"apiVersion": "2021-02-01",
|
|
"name": "[format('{0}/{1}', variables('crPrivateEndpointName_var'), 'default')]",
|
|
"properties": {
|
|
"privateDnsZoneConfigs": [
|
|
{
|
|
"name": "privatelink-azurecr-io",
|
|
"properties": {
|
|
"privateDnsZoneId": "[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameCr_var'))]"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Network/privateEndpoints', variables('crPrivateEndpointName_var'))]",
|
|
"[resourceId('Microsoft.Network/privateDnsZones', variables('privateDnsNameCr_var'))]"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'MLWorkspace')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "DatabricksCluster",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"location": {
|
|
"value": "[parameters('location')]"
|
|
},
|
|
"identity": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ManagedIdentity'), '2019-10-01').outputs.mIdentityId.value]"
|
|
},
|
|
"adb_workspace_url": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_workspaceUrl.value]"
|
|
},
|
|
"adb_workspace_id": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace'), '2019-10-01').outputs.databricks_workspace_id.value]"
|
|
},
|
|
"adb_secret_scope_name": {
|
|
"value": "[variables('adbAkvLinkName')]"
|
|
},
|
|
"akv_id": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_id.value]"
|
|
},
|
|
"akv_uri": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_uri.value]"
|
|
},
|
|
"LogAWkspId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics'), '2019-10-01').outputs.logAnalyticsWkspId.value]"
|
|
},
|
|
"LogAWkspKey": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics'), '2019-10-01').outputs.primarySharedKey.value]"
|
|
},
|
|
"storageKey": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.key1.value]"
|
|
},
|
|
"evenHubKey": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHPConnString.value]"
|
|
},
|
|
"eventHubId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHubNameId.value]"
|
|
},
|
|
"deployADBCluster": {
|
|
"value": "[parameters('deployADBCluster')]"
|
|
},
|
|
"adb_pat_lifetime": {
|
|
"value": "[parameters('adb_pat_lifetime')]"
|
|
},
|
|
"adb_cluster_name": {
|
|
"value": "[parameters('adb_cluster_name')]"
|
|
},
|
|
"adb_spark_version": {
|
|
"value": "[parameters('adb_spark_version')]"
|
|
},
|
|
"adb_node_type": {
|
|
"value": "[parameters('adb_node_type')]"
|
|
},
|
|
"adb_num_worker": {
|
|
"value": "[parameters('adb_num_worker')]"
|
|
},
|
|
"adb_auto_terminate_min": {
|
|
"value": "[parameters('adb_auto_terminate_min')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "14732904098596994959"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"location": {
|
|
"type": "string"
|
|
},
|
|
"force_update": {
|
|
"type": "string",
|
|
"defaultValue": "[utcNow()]"
|
|
},
|
|
"identity": {
|
|
"type": "string"
|
|
},
|
|
"akv_id": {
|
|
"type": "string"
|
|
},
|
|
"akv_uri": {
|
|
"type": "string"
|
|
},
|
|
"adb_pat_lifetime": {
|
|
"type": "string"
|
|
},
|
|
"adb_workspace_url": {
|
|
"type": "string"
|
|
},
|
|
"adb_workspace_id": {
|
|
"type": "string"
|
|
},
|
|
"adb_secret_scope_name": {
|
|
"type": "string"
|
|
},
|
|
"adb_cluster_name": {
|
|
"type": "string"
|
|
},
|
|
"adb_spark_version": {
|
|
"type": "string"
|
|
},
|
|
"adb_node_type": {
|
|
"type": "string"
|
|
},
|
|
"adb_num_worker": {
|
|
"type": "string"
|
|
},
|
|
"adb_auto_terminate_min": {
|
|
"type": "string"
|
|
},
|
|
"LogAWkspId": {
|
|
"type": "string"
|
|
},
|
|
"LogAWkspKey": {
|
|
"type": "string"
|
|
},
|
|
"storageKey": {
|
|
"type": "string"
|
|
},
|
|
"evenHubKey": {
|
|
"type": "string"
|
|
},
|
|
"eventHubId": {
|
|
"type": "string"
|
|
},
|
|
"deployADBCluster": {
|
|
"type": "bool"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"condition": "[parameters('deployADBCluster')]",
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "createAdbPATToken",
|
|
"location": "[parameters('location')]",
|
|
"kind": "AzureCLI",
|
|
"identity": {
|
|
"type": "UserAssigned",
|
|
"userAssignedIdentities": {
|
|
"[parameters('identity')]": {}
|
|
}
|
|
},
|
|
"properties": {
|
|
"azCliVersion": "2.26.0",
|
|
"timeout": "PT5M",
|
|
"cleanupPreference": "OnExpiration",
|
|
"retentionInterval": "PT1H",
|
|
"environmentVariables": [
|
|
{
|
|
"name": "ADB_WORKSPACE_URL",
|
|
"value": "[parameters('adb_workspace_url')]"
|
|
},
|
|
{
|
|
"name": "ADB_WORKSPACE_ID",
|
|
"value": "[parameters('adb_workspace_id')]"
|
|
},
|
|
{
|
|
"name": "PAT_LIFETIME",
|
|
"value": "[parameters('adb_pat_lifetime')]"
|
|
}
|
|
],
|
|
"scriptContent": "#/bin/bash -e\npat_token_config=$(jq -n -c \\\n --arg ls \"$PAT_LIFETIME\" \\\n --arg co \"Example Token created by Bicep deployment\" \\\n '{lifetime_seconds: ($ls|tonumber),\n comment: $co}')\n\nadbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)\nazureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)\n\nauthHeader=\"Authorization: Bearer $adbGlobalToken\"\nadbSPMgmtToken=\"X-Databricks-Azure-SP-Management-Token:$azureApiToken\"\nadbResourceId=\"X-Databricks-Azure-Workspace-Resource-Id:$ADB_WORKSPACE_ID\"\njson=$(echo \"$pat_token_config\" | curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" --data-binary \"@-\" \"https://${ADB_WORKSPACE_URL}/api/2.0/token/create\")\n\necho \"$json\" >\"$AZ_SCRIPTS_OUTPUT_PATH\"\n"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('deployADBCluster')]",
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "secretScopeLink",
|
|
"location": "[parameters('location')]",
|
|
"kind": "AzureCLI",
|
|
"identity": {
|
|
"type": "UserAssigned",
|
|
"userAssignedIdentities": {
|
|
"[parameters('identity')]": {}
|
|
}
|
|
},
|
|
"properties": {
|
|
"azCliVersion": "2.26.0",
|
|
"timeout": "PT1H",
|
|
"cleanupPreference": "OnExpiration",
|
|
"retentionInterval": "PT1H",
|
|
"environmentVariables": [
|
|
{
|
|
"name": "ADB_WORKSPACE_URL",
|
|
"value": "[parameters('adb_workspace_url')]"
|
|
},
|
|
{
|
|
"name": "ADB_WORKSPACE_ID",
|
|
"value": "[parameters('adb_workspace_id')]"
|
|
},
|
|
{
|
|
"name": "ADB_SECRET_SCOPE_NAME",
|
|
"value": "[parameters('adb_secret_scope_name')]"
|
|
},
|
|
{
|
|
"name": "AKV_ID",
|
|
"value": "[parameters('akv_id')]"
|
|
},
|
|
{
|
|
"name": "AKV_URI",
|
|
"value": "[parameters('akv_uri')]"
|
|
},
|
|
{
|
|
"name": "ADB_LOG_WKSP_ID",
|
|
"value": "[parameters('LogAWkspId')]"
|
|
},
|
|
{
|
|
"name": "ADB_LOG_WKSP_KEY",
|
|
"value": "[parameters('LogAWkspKey')]"
|
|
},
|
|
{
|
|
"name": "STORAGE_ACCESS_KEY",
|
|
"value": "[parameters('storageKey')]"
|
|
},
|
|
{
|
|
"name": "EVENT_HUB_KEY",
|
|
"value": "[parameters('evenHubKey')]"
|
|
},
|
|
{
|
|
"name": "ADB_PAT_TOKEN",
|
|
"value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', 'createAdbPATToken')).outputs.token_value]"
|
|
}
|
|
],
|
|
"scriptContent": "#/bin/bash -e\n\n# TODO: Create AKV backed secret scope using User assigned managed identity. Currently not supported\n# akv_backed='{\"resource_id\": \"'${AKV_ID}'\",\"dns_name\": \"'${AKV_URI}'\"}'\n# akv_secret_scope_payload=$(\n# jq -n -c \\\n# --arg sc \"akv_test\" \\\n# --arg bak \"$akv_backed\" \\\n# '{\n# scope: $sc,\n# scope_backend_type: \"AZURE_KEYVAULT\",\n# initial_manage_principal: \"users\",\n# backend_azure_keyvault: ($bak|fromjson)\n# }'\n# )\n\n# Create secret scope backed by ADB\nadb_secret_scope_payload=$(\n jq -n -c \\\n --arg sc \"$ADB_SECRET_SCOPE_NAME\" \\\n '{\n scope: $sc,\n initial_manage_principal: \"users\"\n }'\n)\n\nadbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)\nazureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)\n\nauthHeader=\"Authorization: Bearer $adbGlobalToken\"\nadbSPMgmtToken=\"X-Databricks-Azure-SP-Management-Token:$azureApiToken\"\nadbResourceId=\"X-Databricks-Azure-Workspace-Resource-Id:$ADB_WORKSPACE_ID\"\n\necho \"Create ADB secret scope backed by Databricks key vault\"\njson=$(echo $adb_secret_scope_payload | curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" --data-binary \"@-\" \"https://${ADB_WORKSPACE_URL}/api/2.0/secrets/scopes/create\")\n\nfunction createKey() {\n local keyName=$1\n local secretValue=$2\n json_payload=$(\n jq -n -c \\\n --arg sc \"$ADB_SECRET_SCOPE_NAME\" \\\n --arg k \"$keyName\" \\\n --arg v \"$secretValue\" \\\n '{\n scope: $sc,\n key: $k,\n string_value: $v\n }'\n )\n response=$(echo $json_payload | curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" --data-binary \"@-\" \"https://${ADB_WORKSPACE_URL}/api/2.0/secrets/put\")\n echo $response\n\n}\n\ncreateKey \"LogAWkspId\" \"$ADB_LOG_WKSP_ID\"\ncreateKey \"LogAWkspkey\" \"$ADB_LOG_WKSP_KEY\"\nC_ADB_PAT_TOKEN=$(sed -e 's/[\\\"\\\\]//g' <<<$ADB_PAT_TOKEN)\ncreateKey \"dbPATKey\" \"$C_ADB_PAT_TOKEN\"\ncreateKey \"ehKey\" \"$EVENT_HUB_KEY\"\ncreateKey \"stgAccessKey\" \"$STORAGE_ACCESS_KEY\"\n\necho \"$json\" >\"$AZ_SCRIPTS_OUTPUT_PATH\"\n\n# tail -f /dev/null"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Resources/deploymentScripts', 'createAdbPATToken')]"
|
|
]
|
|
},
|
|
{
|
|
"condition": "[parameters('deployADBCluster')]",
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "uploadFilesToAdb",
|
|
"location": "[parameters('location')]",
|
|
"kind": "AzureCLI",
|
|
"identity": {
|
|
"type": "UserAssigned",
|
|
"userAssignedIdentities": {
|
|
"[parameters('identity')]": {}
|
|
}
|
|
},
|
|
"properties": {
|
|
"azCliVersion": "2.26.0",
|
|
"timeout": "PT5M",
|
|
"cleanupPreference": "OnExpiration",
|
|
"retentionInterval": "PT1H",
|
|
"environmentVariables": [
|
|
{
|
|
"name": "ADB_WORKSPACE_URL",
|
|
"value": "[parameters('adb_workspace_url')]"
|
|
},
|
|
{
|
|
"name": "ADB_WORKSPACE_ID",
|
|
"value": "[parameters('adb_workspace_id')]"
|
|
},
|
|
{
|
|
"name": "EVENT_HUB_ID",
|
|
"value": "[parameters('eventHubId')]"
|
|
}
|
|
],
|
|
"scriptContent": "#/bin/bash -e\nUSER_FOLDER=$(pwd)\n\nadbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)\nazureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)\n\nauthHeader=\"Authorization: Bearer $adbGlobalToken\"\nadbSPMgmtToken=\"X-Databricks-Azure-SP-Management-Token:$azureApiToken\"\nadbResourceId=\"X-Databricks-Azure-Workspace-Resource-Id:$ADB_WORKSPACE_ID\"\n\necho \"Download init script\"\nmkdir -p init_scripts && cd init_scripts\ncurl -L \\\n -O \"https://raw.githubusercontent.com/lordlinus/databricks-all-in-one-bicep-template/main/databricks/init_scripts/capture_log_metrics.sh\"\ncd $USER_FOLDER\n\necho \"Upload init script to /databricks/init/capture_log_metrics.sh\"\ncurl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" \\\n https://${ADB_WORKSPACE_URL}/api/2.0/dbfs/put \\\n --form contents=@init_scripts/capture_log_metrics.sh \\\n --form path=\"/databricks/init/capture_log_metrics.sh\" \\\n --form overwrite=true\n\necho \"Download Sample notebooks\"\nmkdir -p notebooks && cd notebooks\ncurl -L \\\n -O \"https://raw.githubusercontent.com/lordlinus/databricks-all-in-one-bicep-template/00a2978db789d1f1edf63603666d37a1ab72c86f/databricks/notebooks/azure_runner_docs_example.ipynb\" \\\n -O \"https://raw.githubusercontent.com/lordlinus/databricks-all-in-one-bicep-template/00a2978db789d1f1edf63603666d37a1ab72c86f/databricks/notebooks/timezone_test.ipynb\"\ncd $USER_FOLDER\n\necho \"Upload Sample notebooks\"\nfor notebook in notebooks/*.ipynb; do\n filename=$(basename $notebook)\n echo \"Upload sample notebook $notebook to workspace\"\n curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" \\\n https://${ADB_WORKSPACE_URL}/api/2.0/workspace/import \\\n --form contents=@\"$notebook\" \\\n --form path=\"/Shared/$filename\" \\\n --form format=JUPYTER \\\n --form language=SCALA \\\n --form overwrite=true\ndone\n\necho \"Download Loganalytics jar files\"\nmkdir -p jars && cd jars\ncurl -L \\\n -O \"https://raw.githubusercontent.com/lordlinus/databricks-all-in-one-bicep-template/00a2978db789d1f1edf63603666d37a1ab72c86f/databricks/jars/spark-listeners-loganalytics_3.0.1_2.12-1.0.0.jar\" \\\n -O \"https://raw.githubusercontent.com/lordlinus/databricks-all-in-one-bicep-template/00a2978db789d1f1edf63603666d37a1ab72c86f/databricks/jars/spark-listeners_3.0.1_2.12-1.0.0.jar\"\ncd $USER_FOLDER\n\necho \"Upload jar files\"\nfor jar_file in jars/*.jar; do\n filename=$(basename $jar_file)\n echo \"Upload $jar_file file to DBFS path\"\n curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" \\\n https://${ADB_WORKSPACE_URL}/api/2.0/dbfs/put \\\n --form filedata=@\"$jar_file\" \\\n --form path=\"/FileStore/jars/$filename\" \\\n --form overwrite=true\ndone\n\n# Get ADB log categories\nadb_logs_types=$(az monitor diagnostic-settings categories list --resource $ADB_WORKSPACE_ID | jq -c '.value[] | {category: .name, enabled:true}' | jq --slurp .)\n\n# Enable monitoring for all the categories\nadb_monitoring=$(az monitor diagnostic-settings create \\\n --name sparkmonitor \\\n --event-hub $EVENT_HUB_ID \\\n --event-hub-rule \"RootManageSharedAccessKey\" \\\n --resource $ADB_WORKSPACE_ID \\\n --logs \"$adb_logs_types\")"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('deployADBCluster')]",
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "createAdbCluster",
|
|
"location": "[parameters('location')]",
|
|
"kind": "AzureCLI",
|
|
"identity": {
|
|
"type": "UserAssigned",
|
|
"userAssignedIdentities": {
|
|
"[parameters('identity')]": {}
|
|
}
|
|
},
|
|
"properties": {
|
|
"azCliVersion": "2.26.0",
|
|
"timeout": "PT5M",
|
|
"retentionInterval": "PT1H",
|
|
"cleanupPreference": "OnExpiration",
|
|
"forceUpdateTag": "[parameters('force_update')]",
|
|
"environmentVariables": [
|
|
{
|
|
"name": "ADB_WORKSPACE_URL",
|
|
"value": "[parameters('adb_workspace_url')]"
|
|
},
|
|
{
|
|
"name": "ADB_WORKSPACE_ID",
|
|
"value": "[parameters('adb_workspace_id')]"
|
|
},
|
|
{
|
|
"name": "ADB_SECRET_SCOPE_NAME",
|
|
"value": "[parameters('adb_secret_scope_name')]"
|
|
},
|
|
{
|
|
"name": "DATABRICKS_CLUSTER_NAME",
|
|
"value": "[parameters('adb_cluster_name')]"
|
|
},
|
|
{
|
|
"name": "DATABRICKS_SPARK_VERSION",
|
|
"value": "[parameters('adb_spark_version')]"
|
|
},
|
|
{
|
|
"name": "DATABRICKS_NODE_TYPE",
|
|
"value": "[parameters('adb_node_type')]"
|
|
},
|
|
{
|
|
"name": "DATABRICKS_NUM_WORKERS",
|
|
"value": "[parameters('adb_num_worker')]"
|
|
},
|
|
{
|
|
"name": "DATABRICKS_AUTO_TERMINATE_MINUTES",
|
|
"value": "[parameters('adb_auto_terminate_min')]"
|
|
}
|
|
],
|
|
"scriptContent": "#/bin/bash -e\n\n# Databricks cluster config variables\nDATABRICKS_SPARK_CONF='{\n \"spark.databricks.delta.preview.enabled\": \"true\",\n \"spark.eventLog.unknownRecord.maxSize\":\"16m\"\n }'\nDATABRICKS_INIT_CONFIG='{\n \"dbfs\": {\n \"destination\": \"dbfs:/databricks/init/capture_log_metrics.sh\"\n }\n }'\nDATABRICKS_ENV_VARS='{\n \"LOG_ANALYTICS_WORKSPACE_ID\": \"{{secrets/'$ADB_SECRET_SCOPE_NAME'/LogAWkspId}}\",\n \"LOG_ANALYTICS_WORKSPACE_KEY\": \"{{secrets/'$ADB_SECRET_SCOPE_NAME'/LogAWkspkey}}\"\n }'\nDATABRICKS_CLUSTER_LOG='{\n \"dbfs\": {\n \"destination\": \"dbfs:/logs\"\n }\n}'\n\n# Databricks Auth headers\nadbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)\nazureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)\n\n# Create Auth header for Databricks\nauthHeader=\"Authorization: Bearer $adbGlobalToken\"\nadbSPMgmtToken=\"X-Databricks-Azure-SP-Management-Token:$azureApiToken\"\nadbResourceId=\"X-Databricks-Azure-Workspace-Resource-Id:$ADB_WORKSPACE_ID\"\n\necho \"Create Cluster\"\n\nCLUSTER_CREATE_JSON_STRING=$(jq -n -c \\\n --arg cn \"$DATABRICKS_CLUSTER_NAME\" \\\n --arg sv \"$DATABRICKS_SPARK_VERSION\" \\\n --arg nt \"$DATABRICKS_NODE_TYPE\" \\\n --arg nw \"$DATABRICKS_NUM_WORKERS\" \\\n --arg spc \"$DATABRICKS_SPARK_CONF\" \\\n --arg at \"$DATABRICKS_AUTO_TERMINATE_MINUTES\" \\\n --arg is \"$DATABRICKS_INIT_CONFIG\" \\\n --arg ev \"$DATABRICKS_ENV_VARS\" \\\n --arg cl \"$DATABRICKS_CLUSTER_LOG\" \\\n '{cluster_name: $cn,\n idempotency_token: $cn,\n spark_version: $sv,\n node_type_id: $nt,\n num_workers: ($nw|tonumber),\n autotermination_minutes: ($at|tonumber),\n spark_conf: ($spc|fromjson),\n init_scripts: ($is|fromjson),\n spark_env_vars: ($ev|fromjson),\n cluster_log_conf: ($cl|fromjson)\n }')\n\njson=$(echo $CLUSTER_CREATE_JSON_STRING | curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" --data-binary \"@-\" \"https://${ADB_WORKSPACE_URL}/api/2.0/clusters/create\")\necho \"$json\" >$AZ_SCRIPTS_OUTPUT_PATH\n"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Resources/deploymentScripts', 'secretScopeLink')]",
|
|
"[resourceId('Microsoft.Resources/deploymentScripts', 'uploadFilesToAdb')]"
|
|
]
|
|
},
|
|
{
|
|
"condition": "[parameters('deployADBCluster')]",
|
|
"type": "Microsoft.Resources/deploymentScripts",
|
|
"apiVersion": "2020-10-01",
|
|
"name": "configAdbCluster",
|
|
"location": "[parameters('location')]",
|
|
"kind": "AzureCLI",
|
|
"identity": {
|
|
"type": "UserAssigned",
|
|
"userAssignedIdentities": {
|
|
"[parameters('identity')]": {}
|
|
}
|
|
},
|
|
"properties": {
|
|
"azCliVersion": "2.26.0",
|
|
"timeout": "PT5M",
|
|
"retentionInterval": "PT1H",
|
|
"cleanupPreference": "OnExpiration",
|
|
"forceUpdateTag": "[parameters('force_update')]",
|
|
"environmentVariables": [
|
|
{
|
|
"name": "ADB_WORKSPACE_URL",
|
|
"value": "[parameters('adb_workspace_url')]"
|
|
},
|
|
{
|
|
"name": "ADB_WORKSPACE_ID",
|
|
"value": "[parameters('adb_workspace_id')]"
|
|
},
|
|
{
|
|
"name": "ADB_CLUSTER_ID",
|
|
"value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', 'createAdbCluster')).outputs.cluster_id]"
|
|
}
|
|
],
|
|
"scriptContent": "#/bin/bash -e\n\nadbGlobalToken=$(az account get-access-token --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d --output json | jq -r .accessToken)\nazureApiToken=$(az account get-access-token --resource https://management.core.windows.net/ --output json | jq -r .accessToken)\n\nauthHeader=\"Authorization: Bearer $adbGlobalToken\"\nadbSPMgmtToken=\"X-Databricks-Azure-SP-Management-Token:$azureApiToken\"\nadbResourceId=\"X-Databricks-Azure-Workspace-Resource-Id:$ADB_WORKSPACE_ID\"\n\nlibraries='[\n {\n \"jar\": \"dbfs:/FileStore/jars/spark-listeners_3.0.1_2.12-1.0.0.jar\"\n },\n {\n \"jar\": \"dbfs:/FileStore/jars/spark-listeners-loganalytics_3.0.1_2.12-1.0.0.jar\"\n },\n {\n \"maven\": {\n \"coordinates\": \"com.databricks.labs:overwatch_2.12:0.5.0.4\"\n }\n },\n {\n \"maven\": {\n \"coordinates\": \"com.microsoft.azure:azure-eventhubs-spark_2.12:2.3.18\"\n }\n }\n ]'\n\nlibrary_config=$(\n jq -n -c \\\n --arg aci \"$ADB_CLUSTER_ID\" \\\n --arg li \"$libraries\" \\\n '{\n cluster_id: $aci,\n libraries: ($li|fromjson)\n }'\n)\n\njson=$(echo $library_config | curl -sS -X POST -H \"$authHeader\" -H \"$adbSPMgmtToken\" -H \"$adbResourceId\" --data-binary \"@-\" \"https://${ADB_WORKSPACE_URL}/api/2.0/libraries/install\")\n\necho \"$json\" >\"$AZ_SCRIPTS_OUTPUT_PATH\"\n\n# echo \"Create Overwatch Job\"\n# JOB_CREATE_JSON_STRING=$(jq -n -c \\\n# --arg ci \"$cluster_id\" \\\n# '{name: \"overwatch-job\",\n# existing_cluster_id: $ci,\n# notebook_task: {\n# \"notebook_path\": \"/Shared/azure_runner_docs_example.ipynb\"\n# }\n# }')\n# create_notebook_job=$(echo $JOB_CREATE_JSON_STRING | d_curl \"https://${adbWorkspaceUrl}/api/2.0/jobs/create\")\n# echo $create_notebook_job\n\necho \"Configuring services done\"\n"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Resources/deploymentScripts', 'createAdbCluster')]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"patToken": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.Resources/deploymentScripts', 'createAdbPATToken')).outputs.token_value]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksWorkspace')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'ManagedIdentity')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "CreateAksCluster",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"aksSubnetId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets'), '2019-10-01').outputs.aksSubnet_id.value]"
|
|
},
|
|
"aksAgentCount": {
|
|
"value": "[parameters('aksAgentCount')]"
|
|
},
|
|
"aksAgentVMSize": {
|
|
"value": "[parameters('aksAgentVMSize')]"
|
|
},
|
|
"dnsPrefix": {
|
|
"value": "[variables('aksDNSPrefix')]"
|
|
},
|
|
"clusterName": {
|
|
"value": "[variables('aksAmlComputeName')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "1743251093094924688"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"aksAgentCount": {
|
|
"type": "int"
|
|
},
|
|
"aksAgentVMSize": {
|
|
"type": "string"
|
|
},
|
|
"aksSubnetId": {
|
|
"type": "string"
|
|
},
|
|
"dnsPrefix": {
|
|
"type": "string"
|
|
},
|
|
"maxPods": {
|
|
"type": "int",
|
|
"defaultValue": 110
|
|
},
|
|
"clusterName": {
|
|
"type": "string",
|
|
"metadata": {
|
|
"description": "The name of the Managed Cluster resource."
|
|
}
|
|
},
|
|
"location": {
|
|
"type": "string",
|
|
"defaultValue": "[resourceGroup().location]"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.ContainerService/managedClusters",
|
|
"apiVersion": "2021-07-01",
|
|
"name": "[parameters('clusterName')]",
|
|
"location": "[parameters('location')]",
|
|
"identity": {
|
|
"type": "SystemAssigned"
|
|
},
|
|
"properties": {
|
|
"dnsPrefix": "[parameters('dnsPrefix')]",
|
|
"networkProfile": {
|
|
"networkPlugin": "kubenet",
|
|
"loadBalancerSku": "standard",
|
|
"dnsServiceIP": "10.0.0.10",
|
|
"dockerBridgeCidr": "172.17.0.1/16",
|
|
"podCidr": "10.244.0.0/16"
|
|
},
|
|
"agentPoolProfiles": [
|
|
{
|
|
"name": "agentpool",
|
|
"count": "[parameters('aksAgentCount')]",
|
|
"vmSize": "[parameters('aksAgentVMSize')]",
|
|
"type": "VirtualMachineScaleSets",
|
|
"mode": "System",
|
|
"osType": "Linux",
|
|
"vnetSubnetID": "[parameters('aksSubnetId')]",
|
|
"maxPods": "[parameters('maxPods')]"
|
|
}
|
|
],
|
|
"nodeResourceGroup": "[format('MC_{0}_{1}_{2}', resourceGroup().name, parameters('clusterName'), parameters('location'))]"
|
|
}
|
|
}
|
|
],
|
|
"outputs": {
|
|
"aksClusterResourceId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))]"
|
|
},
|
|
"kubeletidentity": {
|
|
"type": "string",
|
|
"value": "[reference(resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))).identityProfile.kubeletidentity.objectId]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'HubandSpokeVnets')]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "MLWorkspace",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"amlWorkspaceName": {
|
|
"value": "[variables('amlWorkspaceName')]"
|
|
},
|
|
"containerRegistryName": {
|
|
"value": "[variables('containerRegistryName')]"
|
|
},
|
|
"keyVaultIdentifierId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault'), '2019-10-01').outputs.keyvault_id.value]"
|
|
},
|
|
"storageAccount": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.storageaccount_id.value]"
|
|
},
|
|
"applicationInsightsName": {
|
|
"value": "[variables('applicationInsightsName')]"
|
|
},
|
|
"aksClusterId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'CreateAksCluster'), '2019-10-01').outputs.aksClusterResourceId.value]"
|
|
},
|
|
"aksAmlComputeName": {
|
|
"value": "[variables('aksAmlComputeName')]"
|
|
},
|
|
"aksClusterPrincipleId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'CreateAksCluster'), '2019-10-01').outputs.kubeletidentity.value]"
|
|
},
|
|
"sslLeafName": {
|
|
"value": "[variables('sslLeafName')]"
|
|
},
|
|
"linkAkstoAml": {
|
|
"value": "[parameters('linkAkstoAml')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "11483396000871316490"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"keyVaultIdentifierId": {
|
|
"type": "string"
|
|
},
|
|
"storageAccount": {
|
|
"type": "string"
|
|
},
|
|
"amlWorkspaceName": {
|
|
"type": "string"
|
|
},
|
|
"containerRegistryName": {
|
|
"type": "string"
|
|
},
|
|
"applicationInsightsName": {
|
|
"type": "string"
|
|
},
|
|
"aksClusterPrincipleId": {
|
|
"type": "string"
|
|
},
|
|
"aksAmlComputeName": {
|
|
"type": "string"
|
|
},
|
|
"aksClusterId": {
|
|
"type": "string"
|
|
},
|
|
"sslLeafName": {
|
|
"type": "string"
|
|
},
|
|
"linkAkstoAml": {
|
|
"type": "bool"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"variables": {
|
|
"acrPullRoleDefId": "7f951dda-4ed3-4680-a7ca-43fe172d538d",
|
|
"location": "[resourceGroup().location]"
|
|
},
|
|
"resources": [
|
|
{
|
|
"type": "Microsoft.ContainerRegistry/registries",
|
|
"apiVersion": "2021-06-01-preview",
|
|
"name": "[parameters('containerRegistryName')]",
|
|
"location": "[variables('location')]",
|
|
"sku": {
|
|
"name": "Premium"
|
|
},
|
|
"properties": {
|
|
"anonymousPullEnabled": false,
|
|
"dataEndpointEnabled": true,
|
|
"adminUserEnabled": true,
|
|
"publicNetworkAccess": "Enabled",
|
|
"networkRuleBypassOptions": "AzureServices"
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.Authorization/roleAssignments",
|
|
"apiVersion": "2020-08-01-preview",
|
|
"scope": "[format('Microsoft.ContainerRegistry/registries/{0}', parameters('containerRegistryName'))]",
|
|
"name": "[parameters('aksClusterPrincipleId')]",
|
|
"properties": {
|
|
"principalId": "[parameters('aksClusterPrincipleId')]",
|
|
"roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('acrPullRoleDefId'))]"
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Insights/components",
|
|
"apiVersion": "2018-05-01-preview",
|
|
"name": "[parameters('applicationInsightsName')]",
|
|
"location": "[variables('location')]",
|
|
"kind": "web",
|
|
"properties": {
|
|
"Application_Type": "web"
|
|
}
|
|
},
|
|
{
|
|
"type": "Microsoft.MachineLearningServices/workspaces",
|
|
"apiVersion": "2021-04-01",
|
|
"name": "[parameters('amlWorkspaceName')]",
|
|
"location": "[variables('location')]",
|
|
"identity": {
|
|
"type": "SystemAssigned"
|
|
},
|
|
"properties": {
|
|
"friendlyName": "[parameters('amlWorkspaceName')]",
|
|
"storageAccount": "[parameters('storageAccount')]",
|
|
"keyVault": "[parameters('keyVaultIdentifierId')]",
|
|
"applicationInsights": "[resourceId('Microsoft.Insights/components', parameters('applicationInsightsName'))]",
|
|
"containerRegistry": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]",
|
|
"allowPublicAccessWhenBehindVnet": true
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.Insights/components', parameters('applicationInsightsName'))]",
|
|
"[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]"
|
|
]
|
|
},
|
|
{
|
|
"condition": "[parameters('linkAkstoAml')]",
|
|
"type": "Microsoft.MachineLearningServices/workspaces/computes",
|
|
"apiVersion": "2021-07-01",
|
|
"name": "[format('{0}/{1}', parameters('amlWorkspaceName'), parameters('aksAmlComputeName'))]",
|
|
"location": "[variables('location')]",
|
|
"properties": {
|
|
"computeType": "AKS",
|
|
"computeLocation": "[variables('location')]",
|
|
"resourceId": "[parameters('aksClusterId')]",
|
|
"description": "External AKS cluster to provide inference REST endpoint",
|
|
"properties": {
|
|
"clusterPurpose": "FastProd",
|
|
"sslConfiguration": {
|
|
"status": "Auto",
|
|
"leafDomainLabel": "[parameters('sslLeafName')]",
|
|
"overwriteExistingDomain": true
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[resourceId('Microsoft.MachineLearningServices/workspaces', parameters('amlWorkspaceName'))]"
|
|
]
|
|
}
|
|
],
|
|
"outputs": {
|
|
"amlId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.MachineLearningServices/workspaces', parameters('amlWorkspaceName'))]"
|
|
},
|
|
"amlWkspName": {
|
|
"type": "string",
|
|
"value": "[parameters('amlWorkspaceName')]"
|
|
},
|
|
"amlProperties": {
|
|
"type": "object",
|
|
"value": "[reference(resourceId('Microsoft.MachineLearningServices/workspaces', parameters('amlWorkspaceName')))]"
|
|
},
|
|
"ctrRegistryName": {
|
|
"type": "string",
|
|
"value": "[parameters('containerRegistryName')]"
|
|
},
|
|
"ctrRegistryId": {
|
|
"type": "string",
|
|
"value": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('containerRegistryName'))]"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'CreateAksCluster')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'KeyVault')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
},
|
|
{
|
|
"type": "Microsoft.Resources/deployments",
|
|
"apiVersion": "2019-10-01",
|
|
"name": "UpdateSecretsKeyVault",
|
|
"resourceGroup": "[variables('resourceGroupName')]",
|
|
"properties": {
|
|
"expressionEvaluationOptions": {
|
|
"scope": "inner"
|
|
},
|
|
"mode": "Incremental",
|
|
"parameters": {
|
|
"EventHubPK": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub'), '2019-10-01').outputs.eHPConnString.value]"
|
|
},
|
|
"keyVaultName": {
|
|
"value": "[variables('keyVaultName')]"
|
|
},
|
|
"LogAWkspId": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics'), '2019-10-01').outputs.logAnalyticsWkspId.value]"
|
|
},
|
|
"LogAWkspkey": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics'), '2019-10-01').outputs.primarySharedKey.value]"
|
|
},
|
|
"StorageAccountKey1": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.key1.value]"
|
|
},
|
|
"StorageAccountKey2": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount'), '2019-10-01').outputs.key2.value]"
|
|
},
|
|
"DbPATKey": {
|
|
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksCluster'), '2019-10-01').outputs.patToken.value]"
|
|
},
|
|
"updateAKVKeys": {
|
|
"value": "[parameters('updateAKVKeys')]"
|
|
}
|
|
},
|
|
"template": {
|
|
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
|
|
"contentVersion": "1.0.0.0",
|
|
"metadata": {
|
|
"_generator": {
|
|
"name": "bicep",
|
|
"version": "0.4.613.9944",
|
|
"templateHash": "417945022343519357"
|
|
}
|
|
},
|
|
"parameters": {
|
|
"keyVaultName": {
|
|
"type": "string"
|
|
},
|
|
"LogAWkspId": {
|
|
"type": "string"
|
|
},
|
|
"LogAWkspkey": {
|
|
"type": "secureString"
|
|
},
|
|
"StorageAccountKey1": {
|
|
"type": "secureString"
|
|
},
|
|
"StorageAccountKey2": {
|
|
"type": "secureString"
|
|
},
|
|
"EventHubPK": {
|
|
"type": "secureString"
|
|
},
|
|
"DbPATKey": {
|
|
"type": "secureString"
|
|
},
|
|
"updateAKVKeys": {
|
|
"type": "bool"
|
|
}
|
|
},
|
|
"functions": [],
|
|
"resources": [
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'LogAWkspId')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('LogAWkspId')]"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'LogAWkspkey')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('LogAWkspkey')]"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'StorageAccountKey1')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('StorageAccountKey1')]"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'StorageAccountKey2')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('StorageAccountKey2')]"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'EventHubPK')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('EventHubPK')]"
|
|
}
|
|
},
|
|
{
|
|
"condition": "[parameters('updateAKVKeys')]",
|
|
"type": "Microsoft.KeyVault/vaults/secrets",
|
|
"apiVersion": "2021-04-01-preview",
|
|
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'DbPATKey')]",
|
|
"properties": {
|
|
"contentType": "text/plain",
|
|
"value": "[parameters('DbPATKey')]"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"dependsOn": [
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'StorageAccount')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'DatabricksCluster')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'EventHub')]",
|
|
"[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('resourceGroupName')), 'Microsoft.Resources/deployments', 'LogAnalytics')]",
|
|
"[subscriptionResourceId('Microsoft.Resources/resourceGroups', variables('resourceGroupName'))]"
|
|
]
|
|
}
|
|
]
|
|
} |