{ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "clusterName": { "type": "string", "minLength": 3, "maxLength": 15, "metadata": { "description": "The name of the HPC cluster, must be Azure subscription wide unique. It must contain between 3 and 15 characters with lowercase letters and numbers, and must start with a letter." } }, "sqlServerVMName": { "type": "string", "minLength": 3, "maxLength": 15, "metadata": { "description": "The name of the SQL Server VM." } }, "sqlServerDiskType": { "type": "string", "defaultValue": "Premium_SSD", "allowedValues": [ "Standard_HDD", "Standard_SSD", "Premium_SSD" ], "metadata": { "description": "The disk type of SQL server VM. Note that Premium_SSD only supports some VM sizes, see Azure VM Sizes" } }, "sqlServerVMSize": { "type": "string", "defaultValue": "Standard_DS4_v2", "metadata": { "description": "The VM size for the SQL Server VM, all available VM sizes in Azure can be found at https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes" } }, "headNodeList": { "type": "string", "defaultValue": "HPCHN01,HPCHN02", "minLength": 5, "maxLength": 31, "metadata": { "description": "The list of two head node names separated by comma. Do not add any surrounding whitespace." } }, "headNodeOS": { "type": "string", "defaultValue": "WindowsServer2019", "allowedValues": [ "WindowsServer2016", "WindowsServer2019", "CustomImage" ], "metadata": { "description": "The operating system of the head nodes." } }, "headNodeImageResourceId": { "type": "string", "defaultValue": "/subscriptions/xxx/resourceGroups/xxx/providers/Microsoft.Compute/images/xxx", "metadata": { "description": "Specify only when 'CustomImage' selected for headNodeOS. The resource Id of the head node image, it can be a managed VM image in your own subscription (/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroupName>/providers/Microsoft.Compute/images/<ImageName>) or a shared VM image from Azure Shared Image Gallery (/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroupName>/providers/Microsoft.Compute/galleries/<GalleryName>/images/<ImageName>/versions/<ImageVersion>)." } }, "headNodeOsDiskType": { "type": "string", "defaultValue": "Premium_SSD", "allowedValues": [ "Standard_HDD", "Standard_SSD", "Premium_SSD" ], "metadata": { "description": "The disk type of head node VM. Note that Premium_SSD only supports some VM sizes, see Azure VM Sizes" } }, "headNodeVMSize": { "type": "string", "defaultValue": "Standard_DS4_v2", "metadata": { "description": "The VM size of the head node, all available VM sizes in Azure can be found at Azure VM Sizes. Note that some VM sizes in the list are only available in some particular locations. Please check the availability and the price of the VM sizes at https://azure.microsoft.com/pricing/details/virtual-machines/windows/ before deployment." } }, "computeNodeImage": { "type": "string", "defaultValue": "CentOS_7.9", "allowedValues": [ "CentOS_7.6", "CentOS_7.7", "CentOS_7.8", "CentOS_7.9", "CentOS_7.6_HPC", "CentOS_7.7_HPC", "CentOS_7.8_HPC", "CentOS_7.9_HPC", "RHEL_7.7", "RHEL_7.8", "RHEL_7.9", "RHEL_8.5", "SLES_12_SP5", "SLES_12_SP5_HPC", "SLES_15_SP3", "SLES_15_SP3_HPC", "Ubuntu_16.04", "Ubuntu_18.04", "CentOS_7.6_Gen2", "CentOS_7.7_Gen2", "CentOS_7.8_Gen2", "CentOS_7.9_Gen2", "CentOS_7.6_HPC_Gen2", "CentOS_7.7_HPC_Gen2", "CentOS_7.8_HPC_Gen2", "CentOS_7.9_HPC_Gen2", "RHEL_7.7_Gen2", "RHEL_7.8_Gen2", "RHEL_7.9_Gen2", "RHEL_8.5_Gen2", "SLES_12_SP5_Gen2", "SLES_12_SP5_HPC_Gen2", "SLES_15_SP3_Gen2", "SLES_15_SP3_HPC_Gen2", "Ubuntu_16.04_Gen2", "Ubuntu_18.04_Gen2", "Ubuntu_18.04_HPC_Gen2", "Ubuntu_20.04_HPC_Gen2", "CustomImage" ], "metadata": { "description": "The Linux VM image of the compute nodes" } }, "computeNodeImageResourceId": { "type": "string", "defaultValue": "/subscriptions/xxx/resourceGroups/xxx/providers/Microsoft.Compute/images/xxx", "metadata": { "description": "Specify only when 'CustomImage' selected for computeNodeImage. The resource Id of the compute node image, it can be a managed VM image in your own subscription (/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroupName>/providers/Microsoft.Compute/images/<ImageName>) or a shared VM image from Azure Shared Image Gallery (/subscriptions/<SubscriptionId>/resourceGroups/<ResourceGroupName>/providers/Microsoft.Compute/galleries/<GalleryName>/images/<ImageName>/versions/<ImageVersion>)." } }, "computeNodeNamePrefix": { "type": "string", "defaultValue": "IaaSLnxCN", "minLength": 1, "maxLength": 12, "metadata": { "description": "The name prefix of the compute nodes. It must be no more than 12 characters, begin with a letter, and contain only letters, numbers and hyphens. For example, if 'IaaSCN' is specified, the compute node names will be 'IaaSCN000', 'IaaSCN001', ..." } }, "computeNodeNumber": { "type": "int", "defaultValue": 10, "minValue": 1, "maxValue": 500, "metadata": { "description": "The number of the compute nodes." } }, "computeNodeOsDiskType": { "type": "string", "defaultValue": "Standard_HDD", "allowedValues": [ "Standard_HDD", "Standard_SSD", "Premium_SSD" ], "metadata": { "description": "The disk type of compute node VM. Note that Premium_SSD only supports some VM sizes, see Azure VM Sizes" } }, "computeNodeVMSize": { "type": "string", "defaultValue": "Standard_D3_v2", "metadata": { "description": "The VM size of the compute nodes, all available VM sizes in Azure can be found at Azure VM Sizes. Note that some VM sizes in the list are only available in some particular locations. Please check the availability and the price of the VM sizes at https://azure.microsoft.com/pricing/details/virtual-machines/windows/ before deployment." } }, "computeNodeInAVSet": { "type": "string", "defaultValue": "Auto", "allowedValues": [ "Yes", "No", "Auto" ], "metadata": { "description": "Specify whether you want to create the compute nodes in Azure availability set, if 'Auto' is specified, compute nodes are created in availability set only when the VM size is RDMA capable." } }, "adminUsername": { "type": "string", "defaultValue": "hpcadmin", "metadata": { "description": "Administrator user name for the virtual machines." } }, "adminPassword": { "type": "securestring", "metadata": { "description": "Administrator password for the virtual machines." } }, "sshPublicKey": { "type": "string", "defaultValue": "", "metadata": { "description": "Specify the SSH public key for the Linux nodes if you want to use SSH Key pair to authenticate. If not specified, you can use the adminPassword to authenticate." } }, "sqlLoginName": { "type": "string", "defaultValue": "hpcsql", "metadata": { "description": "Sql login name." } }, "sqlLoginPassword": { "type": "securestring", "metadata": { "description": "Sql login password." } }, "vaultName": { "type": "string", "metadata": { "description": "Name of the KeyVault in which the certificate is stored." } }, "vaultResourceGroup": { "type": "string", "metadata": { "description": "Resource Group of the KeyVault in which the certificate is stored." } }, "certificateUrl": { "type": "string", "metadata": { "description": "Url of the certificate with version in KeyVault e.g. https://testault.vault.azure.net/secrets/testcert/b621es1db241e56a72d037479xab1r7." } }, "certThumbprint": { "type": "string", "metadata": { "description": "Thumbprint of the certificate." } }, "enableManagedIdentityOnHeadNode": { "type": "string", "defaultValue": "Yes", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Specify whether to enable system-assigned managed identity on the head node, and use it to manage the Azure IaaS compute nodes." } }, "createPublicIPAddressForHeadNode": { "type": "string", "defaultValue": "Yes", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Indicates whether to create a public IP address for head nodes." } }, "enableAcceleratedNetworking": { "type": "string", "defaultValue": "No", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Specify whether to create the Azure VMs with accelerated networking or not. Note accelerated networking is supported only for some VM sizes and Linux distributions. If you specify it as 'Yes', you must specify accelerated networking supported VM sizes and operating systems for all the VMs in the cluster. More information about accelerated networking please see https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-powershell." } }, "headNodeDataDiskCount": { "type": "int", "defaultValue": 0, "allowedValues": [ 0, 1, 2, 4, 8 ], "metadata": { "description": "The number of data disks attached to the head node VM." } }, "headNodeDataDiskSize": { "type": "int", "defaultValue": 128, "allowedValues": [ 32, 64, 128, 256, 512, 1024, 2048, 4096 ], "metadata": { "description": "The size in GB of each data disk that is attached to the head node VM." } }, "headNodeDataDiskType": { "type": "string", "defaultValue": "Standard_HDD", "allowedValues": [ "Standard_HDD", "Standard_SSD", "Premium_SSD" ], "metadata": { "description": "Head node data disk type. Note that Premium_SSD only supports some VM sizes, see Azure VM Sizes" } }, "computeNodeDataDiskCount": { "type": "int", "defaultValue": 0, "allowedValues": [ 0, 1, 2, 4, 8 ], "metadata": { "description": "The number of data disks attached to the compute node VM." } }, "computeNodeDataDiskSize": { "type": "int", "defaultValue": 128, "allowedValues": [ 32, 64, 128, 256, 512, 1024, 2048, 4096 ], "metadata": { "description": "The size in GB of each data disk that is attached to the compute node VM." } }, "computeNodeDataDiskType": { "type": "string", "defaultValue": "Standard_HDD", "allowedValues": [ "Standard_HDD", "Standard_SSD", "Premium_SSD" ], "metadata": { "description": "Compute node data disk type. Note that Premium_SSD only supports some VM sizes, see Azure VM Sizes" } }, "useVmssForComputeNodes": { "type": "string", "defaultValue": "No", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Specify whether you want to use the experimental feature to create compute nodes as VM scale set. Note that it is not recommended to use this feature in production cluster." } }, "useSpotInstanceForComputeNodes": { "type": "string", "defaultValue": "No", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Specify whether you want to use the experimental feature to create compute nodes as Azure Spot instances. Note that it is not recommended to use this feature in production cluster." } }, "autoInstallInfiniBandDriver": { "type": "string", "defaultValue": "Yes", "allowedValues": [ "Yes", "No" ], "metadata": { "description": "Specify whether you want to install InfiniBandDriver automatically for the VMs with InfiniBand network. This setting is ignored for the VMs without InfiniBand network." } } }, "variables": { "clusterName": "[trim(parameters('clusterName'))]", "vaultName": "[trim(parameters('vaultName'))]", "vaultResourceGroup": "[trim(parameters('vaultResourceGroup'))]", "certThumbprint": "[trim(parameters('certThumbprint'))]", "headNodeList": "[trim(parameters('headNodeList'))]", "sqlServerVMName": "[trim(parameters('sqlServerVMName'))]", "computeNodeNamePrefix": "[trim(parameters('computeNodeNamePrefix'))]", "managedIdentity": { "type": "SystemAssigned" }, "emptyArray": [], "diskTypes": { "Standard_HDD": "Standard_LRS", "Standard_SSD": "StandardSSD_LRS", "Premium_SSD": "Premium_LRS" }, "headNodeDataDisks": { "copy": [ { "name": "hnDataDisks", "count": "[if(equals(parameters('headNodeDataDiskCount'), 0), 1, parameters('headNodeDataDiskCount'))]", "input": { "lun": "[copyIndex('hnDataDisks')]", "createOption": "Empty", "diskSizeGB": "[parameters('headNodeDataDiskSize')]", "managedDisk": { "storageAccountType": "[variables('diskTypes')[parameters('headNodeDataDiskType')]]" } } } ] }, "storageAccountName": "[concat('hpc', uniqueString(resourceGroup().id, variables('clusterName')))]", "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts',variables('storageAccountName'))]", "lbName": "[concat(variables('clusterName'),'-lb')]", "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]", "lbFrontEndIPConfigID": "[concat(variables('lbID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]", "lbPoolID": "[concat(variables('lbID'),'/backendAddressPools/BackendPool1')]", "lbProbeID": "[concat(variables('lbID'),'/probes/tcpProbe')]", "addressPrefix": "10.0.0.0/16", "subnet1Name": "Subnet-1", "subnet1Prefix": "10.0.0.0/22", "headNodes": "[split(variables('headNodeList'), ',')]", "virtualNetworkName": "[concat(variables('clusterName'),'vnet')]", "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnet1Name'))]", "publicIPSuffix": "[uniqueString(resourceGroup().id)]", "publicIPName": "[concat(variables('clusterName'),'publicip')]", "publicIPDNSNameLabel": "[concat(toLower(variables('clusterName')), variables('publicIPSuffix'))]", "publicIPAddressType": "Dynamic", "publicIpAddressId": { "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPName'))]" }, "availabilitySetNameHN": "[concat(variables('clusterName'), '-avset')]", "cnAvailabilitySetName": "[concat(variables('computeNodeNamePrefix'), 'avset')]", "nbrVMPerAvailabilitySet": 200, "cnAvailabilitySetNumber": "[add(div(parameters('computeNodeNumber'), variables('nbrVMPerAvailabilitySet')), 1)]", "uniqueSuffix": "[uniqueString(variables('subnetRef'))]", "uniqueNicSuffix": "[concat('-nic-', variables('uniqueSuffix'))]", "uniquePubNicSuffix": "[concat('-pubnic-', variables('uniqueSuffix'))]", "nsgName": "[concat('hpcnsg-',uniquestring(resourceGroup().id, variables('subnetRef')))]", "rdmaASeries": [ "Standard_A8", "Standard_A9" ], "cnRDMACapable": "[or(contains(variables('rdmaASeries'), parameters('computeNodeVMSize')), contains(toLower(split(parameters('computeNodeVMSize'), '_')[1]), 'r'))]", "hnRDMACapable": "[or(contains(variables('rdmaASeries'), parameters('headNodeVMSize')), contains(toLower(split(parameters('headNodeVMSize'), '_')[1]), 'r'))]", "createCNInAVSet": "[or(equals(parameters('computeNodeInAVSet'), 'Yes'), and(equals(parameters('computeNodeInAVSet'), 'Auto'), variables('cnRDMACapable')))]", "useVmssForCN": "[equals(parameters('useVmssForComputeNodes'), 'Yes')]", "autoEnableInfiniBand": "[equals(parameters('autoInstallInfiniBandDriver'), 'Yes')]", "vmPriority": "[if(equals(parameters('useSpotInstanceForComputeNodes'), 'Yes'), 'Spot', 'Regular')]", "computeVmssName": "[take(replace(variables('computeNodeNamePrefix'), '-', ''), 9)]", "vmssSinglePlacementGroup": "[lessOrEquals(parameters('computeNodeNumber'), 100)]", "winCertSecrets": [ { "sourceVault": { "id": "[resourceId(variables('vaultResourceGroup'), 'Microsoft.KeyVault/vaults', variables('vaultName'))]" }, "vaultCertificates": [ { "certificateUrl": "[parameters('certificateUrl')]", "certificateStore": "My" } ] } ], "lnxCertSecrets": [ { "sourceVault": { "id": "[resourceId(variables('vaultResourceGroup'), 'Microsoft.KeyVault/vaults', variables('vaultName'))]" }, "vaultCertificates": [ { "certificateUrl": "[parameters('certificateUrl')]" } ] } ], "headNodeImages": { "WindowsServer2016": { "publisher": "MicrosoftWindowsServerHPCPack", "offer": "WindowsServerHPCPack", "sku": "2019hn-ws2016", "version": "latest" }, "WindowsServer2019": { "publisher": "MicrosoftWindowsServerHPCPack", "offer": "WindowsServerHPCPack", "sku": "2019hn-ws2019", "version": "6.0.7531" }, "CustomImage": { "id": "[trim(parameters('headNodeImageResourceId'))]" } }, "computeNodeImages": { "AlmaLinux_8.5": { "publisher": "almalinux", "offer": "almalinux", "sku": "8_5", "version": "latest" }, "AlmaLinux_8.5_HPC": { "publisher": "almalinux", "offer": "almalinux-hpc", "sku": "8_5", "version": "latest" }, "CentOS_7.6": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7.6", "version": "latest" }, "CentOS_7.7": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7.7", "version": "latest" }, "CentOS_7.8": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_8", "version": "latest" }, "CentOS_7.9": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_9", "version": "latest" }, "CentOS_7.6_HPC": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7.6", "version": "latest" }, "CentOS_7.7_HPC": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7.7", "version": "latest" }, "CentOS_7.8_HPC": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_8", "version": "latest" }, "CentOS_7.9_HPC": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_9", "version": "latest" }, "CentOS_7.6_Gen2": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_6-gen2", "version": "latest" }, "CentOS_7.7_Gen2": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_7-gen2", "version": "latest" }, "CentOS_7.8_Gen2": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_8-gen2", "version": "latest" }, "CentOS_7.9_Gen2": { "publisher": "OpenLogic", "offer": "CentOS", "sku": "7_9-gen2", "version": "latest" }, "CentOS_7.6_HPC_Gen2": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_6gen2", "version": "latest" }, "CentOS_7.7_HPC_Gen2": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_7-gen2", "version": "latest" }, "CentOS_7.8_HPC_Gen2": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_8-gen2", "version": "latest" }, "CentOS_7.9_HPC_Gen2": { "publisher": "OpenLogic", "offer": "CentOS-HPC", "sku": "7_9-gen2", "version": "latest" }, "RHEL_7.7": { "publisher": "RedHat", "offer": "RHEL", "sku": "7.7", "version": "latest" }, "RHEL_7.8": { "publisher": "RedHat", "offer": "RHEL", "sku": "7.8", "version": "latest" }, "RHEL_7.9": { "publisher": "RedHat", "offer": "RHEL", "sku": "7_9", "version": "latest" }, "RHEL_8.5": { "publisher": "RedHat", "offer": "RHEL", "sku": "8_5", "version": "latest" }, "RHEL_7.7_Gen2": { "publisher": "RedHat", "offer": "RHEL", "sku": "77-gen2", "version": "latest" }, "RHEL_7.8_Gen2": { "publisher": "RedHat", "offer": "RHEL", "sku": "78-gen2", "version": "latest" }, "RHEL_7.9_Gen2": { "publisher": "RedHat", "offer": "RHEL", "sku": "79-gen2", "version": "latest" }, "RHEL_8.5_Gen2": { "publisher": "RedHat", "offer": "RHEL", "sku": "85-gen2", "version": "latest" }, "SLES_12_SP5": { "publisher": "SUSE", "offer": "sles-12-sp5", "sku": "gen1", "version": "latest" }, "SLES_12_SP5_HPC": { "publisher": "SUSE", "offer": "sles-12-sp5-hpc", "sku": "gen1", "version": "latest" }, "SLES_15_SP3": { "publisher": "SUSE", "offer": "sles-15-sp3", "sku": "gen1", "version": "latest" }, "SLES_15_SP3_HPC": { "publisher": "SUSE", "offer": "sles-15-sp3-hpc", "sku": "gen1", "version": "latest" }, "SLES_12_SP5_Gen2": { "publisher": "SUSE", "offer": "sles-12-sp5", "sku": ".04_Gen2", "version": "latest" }, "SLES_12_SP5_HPC_Gen2": { "publisher": "SUSE", "offer": "sles-12-sp5-hpc", "sku": "gen2", "version": "latest" }, "SLES_15_SP3_Gen2": { "publisher": "SUSE", "offer": "sles-15-sp3", "sku": "gen2", "version": "latest" }, "SLES_15_SP3_HPC_Gen2": { "publisher": "SUSE", "offer": "sles-15-sp3-hpc", "sku": "gen2", "version": "latest" }, "Ubuntu_16.04": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "16.04-LTS", "version": "latest" }, "Ubuntu_18.04": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "18.04-LTS", "version": "latest" }, "Ubuntu_16.04_Gen2": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "16_04-lts-gen2", "version": "latest" }, "Ubuntu_18.04_Gen2": { "publisher": "Canonical", "offer": "UbuntuServer", "sku": "18_04-lts-gen2", "version": "latest" }, "Ubuntu_18.04_HPC_Gen2": { "publisher": "Microsoft-DSVM", "offer": "Ubuntu-HPC", "sku": "1804", "version": "latest" }, "Ubuntu_20.04_HPC_Gen2": { "publisher": "Microsoft-DSVM", "offer": "Ubuntu-HPC", "sku": "2004", "version": "latest" }, "CustomImage": { "id": "[trim(parameters('computeNodeImageResourceId'))]" } }, "headNodeImageRef": "[variables('headNodeImages')[parameters('headNodeOS')]]", "computeNodeImageRef": "[variables('computeNodeImages')[parameters('computeNodeImage')]]", "rdmaDriverSupportedCNImage": "[and(or(contains(parameters('computeNodeImage'), 'CentOS_7'), contains(parameters('computeNodeImage'), 'RHEL_7')), not(contains(parameters('computeNodeImage'), '_HPC')))]", "sharedResxBaseUrl": "https://raw.githubusercontent.com/Azure/hpcpack-template/master/HPCPack2019/shared-resources" }, "resources": [ { "type": "Microsoft.Storage/storageAccounts", "name": "[variables('storageAccountName')]", "apiVersion": "2019-04-01", "location": "[resourceGroup().location]", "sku": { "name": "Standard_LRS" }, "kind": "Storage", "properties": {} }, { "apiVersion": "2019-04-01", "type": "Microsoft.Network/virtualNetworks", "name": "[variables('virtualNetworkName')]", "location": "[resourceGroup().location]", "properties": { "addressSpace": { "addressPrefixes": [ "[variables('addressPrefix')]" ] }, "subnets": [ { "name": "[variables('subnet1Name')]", "properties": { "addressPrefix": "[variables('subnet1Prefix')]" } } ] } }, { "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]", "apiVersion": "2019-04-01", "type": "Microsoft.Network/publicIPAddresses", "name": "[variables('publicIPName')]", "location": "[resourceGroup().location]", "sku": { "name": "Basic" }, "properties": { "publicIPAllocationMethod": "[variables('publicIPAddressType')]", "dnsSettings": { "domainNameLabel": "[variables('publicIPDNSNameLabel')]" } } }, { "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]", "type": "Microsoft.Network/networkSecurityGroups", "name": "[variables('nsgName')]", "apiVersion": "2019-04-01", "location": "[resourceGroup().location]", "properties": { "securityRules": [ { "name": "allow-HTTPS", "properties": { "description": "Allow Https", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "443", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1000, "direction": "Inbound" } }, { "name": "allow-RDP", "properties": { "description": "Allow RDP", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "3389", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1010, "direction": "Inbound" } }, { "name": "allow-HPCSession", "properties": { "description": "Allow HPC Session service", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "9090", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1020, "direction": "Inbound" } }, { "name": "allow-HPCBroker", "properties": { "description": "Allow HPC Broker service", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "9087", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1030, "direction": "Inbound" } }, { "name": "allow-HPCBrokerWorker", "properties": { "description": "Allow HPC Broker worker", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "9091", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1040, "direction": "Inbound" } }, { "name": "allow-HPCDataService", "properties": { "description": "Allow HPC Data service", "protocol": "Tcp", "sourcePortRange": "*", "destinationPortRange": "9094 ", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 1050, "direction": "Inbound" } } ] } }, { "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]", "name": "[concat(variables('headNodes')[copyIndex()], 'PublicIp')]", "type": "Microsoft.Network/publicIPAddresses", "apiVersion": "2019-04-01", "location": "[resourceGroup().location]", "copy": { "name": "HNPips", "count": "[length(variables('headNodes'))]" }, "properties": { "publicIPAllocationMethod": "Dynamic", "dnsSettings": { "domainNameLabel": "[toLower(concat(variables('headNodes')[copyIndex()], variables('publicIPSuffix')))]" } } }, { "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]", "apiVersion": "2019-04-01", "name": "[variables('lbName')]", "type": "Microsoft.Network/loadBalancers", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]", "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPName'))]" ], "properties": { "frontendIPConfigurations": [ { "name": "LoadBalancerFrontEnd", "properties": { "publicIPAddress": "[variables('publicIpAddressId')]" } } ], "backendAddressPools": [ { "name": "BackendPool1" } ], "inboundNatRules": [ { "name": "[concat('RDP-', variables('headNodes')[0])]", "properties": { "frontendIPConfiguration": { "id": "[variables('lbFrontEndIPConfigID')]" }, "protocol": "Tcp", "frontendPort": 50001, "backendPort": 3389, "enableFloatingIP": false } }, { "name": "[concat('RDP-', variables('headNodes')[1])]", "properties": { "frontendIPConfiguration": { "id": "[variables('lbFrontEndIPConfigID')]" }, "protocol": "Tcp", "frontendPort": 50002, "backendPort": 3389, "enableFloatingIP": false } } ], "loadBalancingRules": [ { "name": "LBRule", "properties": { "frontendIPConfiguration": { "id": "[variables('lbFrontEndIPConfigID')]" }, "backendAddressPool": { "id": "[variables('lbPoolID')]" }, "protocol": "Tcp", "frontendPort": 443, "backendPort": 443, "enableFloatingIP": false, "idleTimeoutInMinutes": 5, "probe": { "id": "[variables('lbProbeID')]" } } } ], "probes": [ { "name": "tcpProbe", "properties": { "protocol": "Tcp", "port": 5802, "intervalInSeconds": 5, "numberOfProbes": 2 } } ] } }, { "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]", "apiVersion": "2019-04-01", "type": "Microsoft.Network/networkInterfaces", "name": "[concat(variables('headNodes')[copyIndex()], variables('uniquePubNicSuffix'))]", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]", "[concat('Microsoft.Network/networkSecurityGroups/', variables('nsgName'))]", "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]", "[concat('Microsoft.Network/publicIPAddresses/', concat(variables('headNodes')[copyIndex()], 'PublicIp'))]" ], "copy": { "name": "HNNicsWithPubicIP", "count": "[length(variables('headNodes'))]" }, "properties": { "ipConfigurations": [ { "name": "IPConfig", "properties": { "privateIPAllocationMethod": "Dynamic", "subnet": { "id": "[variables('subnetRef')]" }, "loadBalancerBackendAddressPools": [ { "id": "[variables('lbPoolID')]" } ], "loadBalancerInboundNatRules": [ { "id": "[concat(variables('lbID'),'/inboundNatRules/RDP-', variables('headNodes')[copyIndex()])]" } ], "publicIPAddress": { "id": "[resourceId('Microsoft.Network/publicIPAddresses', concat(variables('headNodes')[copyIndex()], 'PublicIp'))]" } } } ], "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]" }, "enableAcceleratedNetworking": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]" } }, { "condition": "[not(equals(parameters('createPublicIPAddressForHeadNode'), 'Yes'))]", "apiVersion": "2019-04-01", "type": "Microsoft.Network/networkInterfaces", "name": "[concat(variables('headNodes')[copyIndex()], variables('uniqueNicSuffix'))]", "location": "[resourceGroup().location]", "copy": { "name": "HNNicsWithoutPublicIP", "count": "[length(variables('headNodes'))]" }, "properties": { "ipConfigurations": [ { "name": "IPConfig", "properties": { "privateIPAllocationMethod": "Dynamic", "subnet": { "id": "[variables('subnetRef')]" } } } ], "enableAcceleratedNetworking": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]" } }, { "type": "Microsoft.Compute/availabilitySets", "name": "[variables('availabilitySetNameHN')]", "apiVersion": "2019-03-01", "sku": { "name": "Aligned" }, "properties": { "platformUpdateDomainCount": 3, "platformFaultDomainCount": 2 }, "location": "[resourceGroup().location]" }, { "apiVersion": "2019-03-01", "type": "Microsoft.Compute/virtualMachines", "name": "[variables('headNodes')[copyIndex()]]", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Compute/availabilitySets/', variables('availabilitySetNameHN'))]", "[concat('Microsoft.Network/networkInterfaces/', variables('headNodes')[copyIndex()], variables('uniqueNicSuffix'))]", "[concat('Microsoft.Network/networkInterfaces/', variables('headNodes')[copyIndex()], variables('uniquePubNicSuffix'))]" ], "copy": { "name": "createHeadNodeCopy", "count": "[length(variables('headNodes'))]" }, "identity": "[if(equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes'), variables('managedIdentity'), json('null'))]", "properties": { "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets', variables('availabilitySetNameHN'))]" }, "hardwareProfile": { "vmSize": "[parameters('headNodeVMSize')]" }, "osProfile": { "computerName": "[variables('headNodes')[copyIndex()]]", "adminUsername": "[parameters('adminUsername')]", "adminPassword": "[parameters('adminPassword')]", "windowsConfiguration": { "enableAutomaticUpdates": false }, "secrets": "[variables('winCertSecrets')]" }, "storageProfile": { "imageReference": "[variables('headNodeImageRef')]", "osDisk": { "name": "[concat(variables('headNodes')[copyIndex()], '-osdisk')]", "caching": "ReadOnly", "createOption": "FromImage", "managedDisk": { "storageAccountType": "[variables('diskTypes')[parameters('headNodeOsDiskType')]]" } }, "dataDisks": "[if(equals(parameters('headNodeDataDiskCount'), 0), variables('emptyArray'), variables('headNodeDataDisks').hnDataDisks)]" }, "networkProfile": { "networkInterfaces": [ { "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(variables('headNodes')[copyIndex()], if(equals(parameters('createPublicIPAddressForHeadNode'), 'Yes'), variables('uniquePubNicSuffix'), variables('uniqueNicSuffix'))))]" } ] } } }, { "condition": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]", "type": "Microsoft.Authorization/roleAssignments", "name": "[guid(resourceGroup().id, variables('clusterName'), variables('headNodes')[copyIndex()])]", "apiVersion": "2017-09-01", "copy": { "name": "msiRoleAssignmentCopy", "count": "[length(variables('headNodes'))]" }, "properties": { "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()]), '2019-03-01', 'Full').identity.principalId]", "scope": "[resourceGroup().id]" }, "dependsOn": [ "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]" ] }, { "condition": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]", "apiVersion": "2019-05-01", "type": "Microsoft.Resources/deployments", "name": "[concat('msiKeyVaultRoleAssignment', copyIndex())]", "copy": { "name": "msiKeyVaultRoleAssignmentCopy", "count": "[length(variables('headNodes'))]" }, "resourceGroup": "[variables('vaultResourceGroup')]", "dependsOn": [ "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]" ], "properties": { "mode": "Incremental", "template": { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "resources": [ { "type": "Microsoft.KeyVault/vaults/providers/roleAssignments", "name": "[concat(variables('vaultName'), '/Microsoft.Authorization/', guid(variables('vaultName'), variables('vnetID'), variables('headNodes')[copyIndex()]))]", "apiVersion": "2017-09-01", "properties": { "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/f25e0fa2-a7c8-4377-a976-54943a77a395", "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()]), '2019-03-01', 'Full').identity.principalId]" } } ] } } }, { "condition": "[and(variables('hnRDMACapable'), variables('autoEnableInfiniBand'))]", "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('headNodes')[copyIndex()], '/installInfiniBandDriver')]", "apiVersion": "2019-03-01", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]" ], "copy": { "name": "installHNInfiniBandDriverCopy", "count": "[length(variables('headNodes'))]" }, "properties": { "publisher": "Microsoft.HpcCompute", "type": "InfiniBandDriverWindows", "typeHandlerVersion": "1.2", "autoUpgradeMinorVersion": true } }, { "apiVersion": "2019-05-01", "type": "Microsoft.Resources/deployments", "name": "[concat('createDBServer', variables('sqlServerVMName'))]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]", "[concat('Microsoft.Compute/availabilitySets/', variables('availabilitySetNameHN'))]" ], "properties": { "mode": "Incremental", "templateLink": { "uri": "[concat(variables('sharedResxBaseUrl'), '/windowsvm-dsc.json')]", "contentVersion": "1.0.0.0" }, "parameters": { "subnetId": { "value": "[variables('subnetRef')]" }, "vmName": { "value": "[variables('sqlServerVMName')]" }, "vmSize": { "value": "[parameters('sqlServerVMSize')]" }, "osDiskType": { "value": "[variables('diskTypes')[parameters('sqlServerDiskType')]]" }, "imageReference": { "value": { "publisher": "MicrosoftSQLServer", "offer": "SQL2016SP2-WS2016", "sku": "Standard", "version": "13.2.220526" } }, "adminUsername": { "value": "[parameters('adminUsername')]" }, "adminPassword": { "value": "[parameters('adminPassword')]" }, "availabilitySetName": { "value": "[variables('availabilitySetNameHN')]" }, "dataDiskCount":{ "value": 1 }, "dataDiskSizeInGB": { "value": 200 }, "dataDiskType": { "value": "[variables('diskTypes')[parameters('sqlServerDiskType')]]" }, "enableAcceleratedNetworking": { "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]" }, "dscExtensionName": { "value": "configSQLServer" }, "dscSettings": { "value": { "configuration": { "url": "[concat(variables('sharedResxBaseUrl'),'/ConfigSQLServer.ps1.zip')]", "script": "ConfigSQLServer.ps1", "function": "ConfigSQLServer" }, "configurationArguments": { } } }, "dscProtectedSettings": { "value": { "configurationArguments": { "LoginCredential": { "UserName": "[parameters('sqlLoginName')]", "Password": "[parameters('sqlLoginPassword')]" } } } } } } }, { "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('headNodes')[0],'/setupPrimaryHeadNode')]", "apiVersion": "2019-03-01", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[0], '/extensions/installInfiniBandDriver')]", "[concat('Microsoft.Resources/deployments/createDBServer', variables('sqlServerVMName'))]", "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]" ], "properties": { "publisher": "Microsoft.Powershell", "type": "DSC", "typeHandlerVersion": "2.80", "autoUpgradeMinorVersion": true, "settings": { "configuration": { "url": "[concat(variables('sharedResxBaseUrl'),'/InstallPrimaryHeadNode.ps1.zip')]", "script": "InstallPrimaryHeadNode.ps1", "function": "InstallPrimaryHeadNode" }, "configurationArguments": { "SSLThumbprint": "[variables('certThumbprint')]", "ClusterName": "[variables('clusterName')]", "SQLServerInstance": "[variables('sqlServerVMName')]", "EnableBuiltinHA": true, "CNSize": "[parameters('computeNodeVMSize')]", "SubscriptionId": "[subscription().subscriptionId]", "VNet": "[variables('virtualNetworkName')]", "Subnet": "[variables('subnet1Name')]", "Location": "[resourceGroup().location]", "ResourceGroup": "[resourceGroup().name]", "VaultResourceGroup": "[variables('vaultResourceGroup')]", "CertificateUrl": "[parameters('CertificateUrl')]", "CNNamePrefix": "[variables('computeNodeNamePrefix')]", "AutoGSUseManagedIdentity": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]" } }, "protectedSettings": { "configurationArguments": { "SetupUserCredential": { "UserName": "[parameters('adminUsername')]", "Password": "[parameters('adminPassword')]" }, "AzureStorageConnString": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';AccountKey=', listKeys(variables('storageAccountId'), '2019-04-01').keys[0].value)]", "SqlLoginCredential": { "UserName": "[parameters('sqlLoginName')]", "Password": "[parameters('sqlLoginPassword')]" } } } } }, { "type": "Microsoft.Compute/virtualMachines/extensions", "name": "[concat(variables('headNodes')[1],'/setupSecondaryHeadNode')]", "apiVersion": "2019-03-01", "location": "[resourceGroup().location]", "dependsOn": [ "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[1], '/extensions/installInfiniBandDriver')]", "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[0], '/extensions/setupPrimaryHeadNode')]" ], "properties": { "publisher": "Microsoft.Powershell", "type": "DSC", "typeHandlerVersion": "2.80", "autoUpgradeMinorVersion": true, "settings": { "configuration": { "url": "[concat(variables('sharedResxBaseUrl'), '/InstallHpcNode.ps1.zip')]", "script": "InstallHpcNode.ps1", "function": "InstallHpcNode" }, "configurationArguments": { "NodeType": "PassiveHeadNode", "HeadNodeList": "[variables('headNodeList')]", "SSLThumbprint": "[variables('certThumbprint')]" } }, "protectedSettings": { "configurationArguments": { "SetupUserCredential": { "UserName": "[parameters('adminUsername')]", "Password": "[parameters('adminPassword')]" } } } } }, { "condition": "[variables('createCNInAVSet')]", "type": "Microsoft.Compute/availabilitySets", "name": "[concat(variables('cnAvailabilitySetName'), padLeft(string(copyIndex()), 2, '0'))]", "apiVersion": "2019-03-01", "location": "[resourceGroup().location]", "sku": { "name": "Aligned" }, "properties": { "platformUpdateDomainCount": 5, "platformFaultDomainCount": 2 }, "copy": { "name": "cnAvailabilitySetCopy", "count": "[variables('cnAvailabilitySetNumber')]" } }, { "condition": "[and(greater(parameters('computeNodeNumber'), 0), not(variables('useVmssForCN')))]", "apiVersion": "2019-05-01", "type": "Microsoft.Resources/deployments", "name": "[concat('create', variables('computeNodeNamePrefix'), padLeft(string(copyIndex()), 3, '0'))]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]", "cnAvailabilitySetCopy" ], "copy": { "name": "CN", "count": "[parameters('computeNodeNumber')]" }, "properties": { "mode": "Incremental", "templateLink": { "uri": "[concat(variables('sharedResxBaseUrl'), '/computenode.json')]", "contentVersion": "1.0.0.0" }, "parameters": { "subnetId": { "value": "[variables('subnetRef')]" }, "vmName": { "value": "[concat(variables('computeNodeNamePrefix'), padLeft(string(copyIndex()), 3, '0'))]" }, "vmSize": { "value": "[parameters('computeNodeVMSize')]" }, "osDiskType": { "value": "[variables('diskTypes')[parameters('computeNodeOsDiskType')]]" }, "dataDiskSizeInGB": { "value": "[parameters('computeNodeDataDiskSize')]" }, "dataDiskCount": { "value": "[parameters('computeNodeDataDiskCount')]" }, "dataDiskType": { "value": "[variables('diskTypes')[parameters('computeNodeDataDiskType')]]" }, "imageReference": { "value": "[variables('computeNodeImageRef')]" }, "imageOsPlatform": { "value": "linux" }, "adminUsername": { "value": "[parameters('adminUsername')]" }, "adminPassword": { "value": "[parameters('adminPassword')]" }, "sshPublicKey": { "value": "[parameters('sshPublicKey')]" }, "availabilitySetName": { "value": "[if(variables('createCNInAVSet'), concat(variables('cnAvailabilitySetName'), padLeft(string(div(copyIndex(), variables('nbrVMPerAvailabilitySet'))), 2, '0')), '')]" }, "vmPriority": { "value": "[variables('vmPriority')]" }, "installRDMADriver": { "value": "[and(variables('cnRDMACapable'), variables('autoEnableInfiniBand'), variables('rdmaDriverSupportedCNImage'))]" }, "enableAcceleratedNetworking": { "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]" }, "secrets": { "value": "[variables('lnxCertSecrets')]" }, "certThumbprint": { "value": "[variables('certThumbprint')]" }, "headNodeList": { "value": "[variables('headNodeList')]" }, "joinDomain": { "value": false }, "domainName": { "value": "" } } } }, { "condition": "[and(greater(parameters('computeNodeNumber'), 0), variables('useVmssForCN'))]", "apiVersion": "2019-05-01", "type": "Microsoft.Resources/deployments", "name": "[concat('create', variables('computeVmssName'))]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]" ], "properties": { "mode": "Incremental", "templateLink": { "uri": "[concat(variables('sharedResxBaseUrl'), '/computevmss.json')]", "contentVersion": "1.0.0.0" }, "parameters": { "subnetId": { "value": "[variables('subnetRef')]" }, "vmssName": { "value": "[variables('computeVmssName')]" }, "vmNumber": { "value": "[parameters('computeNodeNumber')]" }, "vmSize": { "value": "[parameters('computeNodeVMSize')]" }, "osDiskType": { "value": "[variables('diskTypes')[parameters('computeNodeOsDiskType')]]" }, "dataDiskSizeInGB": { "value": "[parameters('computeNodeDataDiskSize')]" }, "dataDiskCount": { "value": "[parameters('computeNodeDataDiskCount')]" }, "dataDiskType": { "value": "[variables('diskTypes')[parameters('computeNodeDataDiskType')]]" }, "imageReference": { "value": "[variables('computeNodeImageRef')]" }, "imageOsPlatform": { "value": "linux" }, "adminUsername": { "value": "[parameters('adminUsername')]" }, "adminPassword": { "value": "[parameters('adminPassword')]" }, "sshPublicKey": { "value": "[parameters('sshPublicKey')]" }, "singlePlacementGroup": { "value": "[variables('vmssSinglePlacementGroup')]" }, "vmPriority": { "value": "[variables('vmPriority')]" }, "installRDMADriver": { "value": "[and(variables('cnRDMACapable'), variables('autoEnableInfiniBand'), variables('rdmaDriverSupportedCNImage'))]" }, "enableAcceleratedNetworking": { "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]" }, "secrets": { "value": "[variables('lnxCertSecrets')]" }, "certThumbprint": { "value": "[variables('certThumbprint')]" }, "headNodeList": { "value": "[variables('headNodeList')]" }, "joinDomain": { "value": false }, "domainName": { "value": "" } } } } ], "outputs": { "clusterDNSName": { "type": "string", "value": "[if(equals(parameters('createPublicIPAddressForHeadNode'), 'No'), '', reference(variables('publicIpAddressId').id, '2019-04-01').dnsSettings.fqdn)]" } } }