hpcpack-template/HPCPack2019-Update1/newcluster-templates/two-hns-wincn-ad.json

{
  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "clusterName": {
      "type": "string",
      "minLength": 3,
      "maxLength": 12,
      "metadata": {
        "description": "The name of the HPC cluster, also used as the host name prefix of the domain controller. It must contain between 3 and 12 characters with lowercase letters and numbers, and must start with a letter."
      }
    },
    "domainName": {
      "type": "string",
      "defaultValue": "hpc.local",
      "metadata": {
        "description": "The fully qualified domain name (FQDN) for the private domain forest which will be created by this template, for example 'hpc.local'."
      }
    },
    "domainControllerVMSize": {
      "type": "string",
      "defaultValue": "Standard_D2_v3",
      "metadata": {
        "description": "The VM size for the domain controller, all available VM sizes in Azure can be found at https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes"
      }
    },
    "sqlServerVMName": {
      "type": "string",
      "minLength": 3,
      "maxLength": 15,
      "metadata": {
        "description": "The name of the SQL Server VM."
      }
    },
    "sqlServerDiskType": {
      "type": "string",
      "defaultValue": "Premium_SSD",
      "allowedValues": [
        "Standard_HDD",
        "Standard_SSD",
        "Premium_SSD"
      ],
      "metadata": {
        "description": "The disk type of SQL server VM. Note that Premium_SSD only supports some VM sizes, see <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>"
      }
    },
    "sqlServerVMSize": {
      "type": "string",
      "defaultValue": "Standard_DS4_v2",
      "metadata": {
        "description": "The VM size for the SQL Server VM, all available VM sizes in Azure can be found at https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes"
      }
    },
    "headNodeList": {
      "type": "string",
      "defaultValue": "HPCHN01,HPCHN02",
      "minLength": 5,
      "maxLength": 31,
      "metadata": {
        "description": "The list of two head node names separated by comma without any surrounding whitespace, for example 'HPCHN01,HPCHN02'. The head node names must be unique in the domain forest."
      }
    },
    "headNodeOS": {
      "type": "string",
      "defaultValue": "WindowsServer2019",
      "allowedValues": [
        "WindowsServer2016",
        "WindowsServer2019",
        "CustomImage"
      ],
      "metadata": {
        "description": "The operating system of the head nodes."
      }
    },
    "headNodeImageResourceId": {
      "type": "string",
      "defaultValue": "/subscriptions/xxx/resourceGroups/xxx/providers/Microsoft.Compute/images/xxx",
      "metadata": {
        "description": "Specify only when 'CustomImage' selected for headNodeOS. The resource Id of the head node image, it can be a managed VM image in your own subscription (/subscriptions/&lt;SubscriptionId&gt;/resourceGroups/&lt;ResourceGroupName&gt;/providers/Microsoft.Compute/images/&lt;ImageName&gt;) or a shared VM image from Azure Shared Image Gallery (/subscriptions/&lt;SubscriptionId&gt;/resourceGroups/&lt;ResourceGroupName&gt;/providers/Microsoft.Compute/galleries/&lt;GalleryName&gt;/images/&lt;ImageName&gt;/versions/&lt;ImageVersion&gt;)."
      }
    },
    "headNodeOsDiskType": {
      "type": "string",
      "defaultValue": "Premium_SSD",
      "allowedValues": [
        "Standard_HDD",
        "Standard_SSD",
        "Premium_SSD"
      ],
      "metadata": {
        "description": "The disk type of head node VM. Note that Premium_SSD only supports some VM sizes, see <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>"
      }
    },
    "headNodeVMSize": {
      "type": "string",
      "defaultValue": "Standard_DS4_v2",
      "metadata": {
        "description": "The VM size of the head node, all available VM sizes in Azure can be found at <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>. Note that some VM sizes in the list are only available in some particular locations. Please check the availability and the price of the VM sizes at https://azure.microsoft.com/pricing/details/virtual-machines/windows/ before deployment."
      }
    },
    "computeNodeNamePrefix": {
      "type": "string",
      "defaultValue": "IaaSCN",
      "minLength": 1,
      "maxLength": 12,
      "metadata": {
        "description": "The name prefix of the compute nodes. It must be no more than 12 characters, begin with a letter, and contain only letters, numbers and hyphens. For example, if 'IaaSCN' is specified, the compute node names will be 'IaaSCN000', 'IaaSCN001', and so on. The compute node names must be unique in the domain forest."
      }
    },
    "computeNodeNumber": {
      "type": "int",
      "defaultValue": 10,
      "minValue": 1,
      "maxValue": 500,
      "metadata": {
        "description": "The number of the compute nodes."
      }
    },
    "computeNodeImage": {
      "type": "string",
      "defaultValue": "WindowsServer2019",
      "allowedValues": [
        "WindowsServer2012",
        "WindowsServer2012R2",
        "WindowsServer2016",
        "WindowsServer2019",
        "WindowsServer2022",
        "WindowsServer2012R2WithExcel",
        "WindowsServer2016WithExcel",
        "WindowsServer2012_Gen2",
        "WindowsServer2012R2_Gen2",
        "WindowsServer2016_Gen2",
        "WindowsServer2019_Gen2",
        "WindowsServer2022_Gen2",
        "CustomImage"
      ],
      "metadata": {
        "description": "The VM image of the compute nodes."
      }
    },
    "computeNodeImageResourceId": {
      "type": "string",
      "defaultValue": "/subscriptions/xxx/resourceGroups/xxx/providers/Microsoft.Compute/images/xxx",
      "metadata": {
        "description": "Specify only when 'CustomImage' selected for computeNodeImage. The resource Id of the compute node image, it can be a managed VM image in your own subscription (/subscriptions/&lt;SubscriptionId&gt;/resourceGroups/&lt;ResourceGroupName&gt;/providers/Microsoft.Compute/images/&lt;ImageName&gt;) or a shared VM image from Azure Shared Image Gallery (/subscriptions/&lt;SubscriptionId&gt;/resourceGroups/&lt;ResourceGroupName&gt;/providers/Microsoft.Compute/galleries/&lt;GalleryName&gt;/images/&lt;ImageName&gt;/versions/&lt;ImageVersion&gt;)."
      }
    },
    "computeNodeOsDiskType": {
      "type": "string",
      "defaultValue": "Standard_HDD",
      "allowedValues": [
        "Standard_HDD",
        "Standard_SSD",
        "Premium_SSD"
      ],
      "metadata": {
        "description": "The disk type of compute node VM. Note that Premium_SSD only supports some VM sizes, see <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>"
      }
    },
    "computeNodeVMSize": {
      "type": "string",
      "defaultValue": "Standard_D3_v2",
      "metadata": {
        "description": "The VM size of the compute nodes, all available VM sizes in Azure can be found at <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>. Note that some VM sizes in the list are only available in some particular locations. Please check the availability and the price of the VM sizes at https://azure.microsoft.com/pricing/details/virtual-machines/windows/ before deployment."
      }
    },
    "computeNodeInAVSet": {
      "type": "string",
      "defaultValue": "Auto",
      "allowedValues": [
        "Yes",
        "No",
        "Auto"
      ],
      "metadata": {
        "description": "Specify whether you want to create the compute nodes in Azure availability set, if 'Auto' is specified, compute nodes are created in availability set only when the VM size is RDMA capable."
      }
    },
    "adminUsername": {
      "type": "string",
      "defaultValue": "hpcadmin",
      "metadata": {
        "description": "Administrator user name for the virtual machines and the Active Directory domain."
      }
    },
    "adminPassword": {
      "type": "securestring",
      "metadata": {
        "description": "Administrator password for the virtual machines and the Active Directory domain"
      }
    },
    "vaultName": {
      "type": "string",
      "metadata": {
        "description": "Name of the KeyVault in which the certificate is stored."
      }
    },
    "vaultResourceGroup": {
      "type": "string",
      "metadata": {
        "description": "Resource Group of the KeyVault in which the certificate is stored."
      }
    },
    "certificateUrl": {
      "type": "string",
      "metadata": {
        "description": "Url of the certificate with version in KeyVault e.g. https://testault.vault.azure.net/secrets/testcert/b621es1db241e56a72d037479xab1r7."
      }
    },
    "certThumbprint": {
      "type": "string",
      "metadata": {
        "description": "Thumbprint of the certificate."
      }
    },
    "enableManagedIdentityOnHeadNode": {
      "type": "string",
      "defaultValue": "Yes",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Specify whether to enable system-assigned managed identity on the head node, and use it to manage the Azure IaaS compute nodes."
      }
    },
    "createPublicIPAddressForHeadNode": {
      "type": "string",
      "defaultValue": "Yes",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Indicates whether to create a public IP address for head nodes."
      }
    },
    "enableAcceleratedNetworking": {
      "type": "string",
      "defaultValue": "No",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Specify whether to create the Azure VMs with accelerated networking or not. Note accelerated networking is supported only for some VM sizes. If you specify it as 'Yes', you must specify accelerated networking supported VM sizes for all the VMs in the cluster. More information about accelerated networking please see https://docs.microsoft.com/en-us/azure/virtual-network/create-vm-accelerated-networking-powershell."
      }
    },
    "headNodeDataDiskCount": {
      "type": "int",
      "defaultValue": 0,
      "allowedValues": [
        0,
        1,
        2,
        4,
        8
      ],
      "metadata": {
        "description": "The number of data disks attached to the head node VM."
      }
    },
    "headNodeDataDiskSize": {
      "type": "int",
      "defaultValue": 128,
      "allowedValues": [
        32,
        64,
        128,
        256,
        512,
        1024,
        2048,
        4096
      ],
      "metadata": {
        "description": "The size in GB of each data disk that is attached to the head node VM."
      }
    },
    "headNodeDataDiskType": {
      "type": "string",
      "defaultValue": "Standard_HDD",
      "allowedValues": [
        "Standard_HDD",
        "Standard_SSD",
        "Premium_SSD"
      ],
      "metadata": {
        "description": "Head node data disk type. Note that Premium_SSD only supports some VM sizes, see <a href='https://azure.microsoft.com/en-us/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>"
      }
    },
    "computeNodeDataDiskCount": {
      "type": "int",
      "defaultValue": 0,
      "allowedValues": [
        0,
        1,
        2,
        4,
        8
      ],
      "metadata": {
        "description": "The number of data disks attached to the compute node VM."
      }
    },
    "computeNodeDataDiskSize": {
      "type": "int",
      "defaultValue": 128,
      "allowedValues": [
        32,
        64,
        128,
        256,
        512,
        1024,
        2048,
        4096
      ],
      "metadata": {
        "description": "The size in GB of each data disk that is attached to the compute node VM."
      }
    },
    "computeNodeDataDiskType": {
      "type": "string",
      "defaultValue": "Standard_HDD",
      "allowedValues": [
        "Standard_HDD",
        "Standard_SSD",
        "Premium_SSD"
      ],
      "metadata": {
        "description": "Compute node data disk type. Note that Premium_SSD only supports some VM sizes, see <a href='https://azure.microsoft.com/documentation/articles/virtual-machines-windows-sizes' target='_blank'>Azure VM Sizes</a>"
      }
    },
    "useVmssForComputeNodes": {
      "type": "string",
      "defaultValue": "No",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Specify whether you want to use the experimental feature to create compute nodes as VM scale set. Note that it is not recommended to use this feature in production cluster."
      }
    },
    "useSpotInstanceForComputeNodes": {
      "type": "string",
      "defaultValue": "No",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Specify whether you want to use the experimental feature to create compute nodes as <a href='https://azure.microsoft.com/pricing/spot/' target='_blank'>Azure Spot instances</a>. Note that it is not recommended to use this feature in production cluster."
      }
    },
    "autoInstallInfiniBandDriver": {
      "type": "string",
      "defaultValue": "Yes",
      "allowedValues": [
        "Yes",
        "No"
      ],
      "metadata": {
        "description": "Specify whether you want to install InfiniBandDriver automatically for the VMs with InfiniBand network. This setting is ignored for the VMs without InfiniBand network."
      }
    }
  },
  "variables": {
    "dcSize": "[trim(parameters('domainControllerVMSize'))]",
    "clusterName": "[trim(parameters('clusterName'))]",
    "domainName": "[trim(parameters('domainName'))]",
    "vaultName": "[trim(parameters('vaultName'))]",
    "vaultResourceGroup": "[trim(parameters('vaultResourceGroup'))]",
    "certThumbprint": "[trim(parameters('certThumbprint'))]",
    "headNodeList": "[trim(parameters('headNodeList'))]",
    "sqlServerVMName": "[trim(parameters('sqlServerVMName'))]",
    "computeNodeNamePrefix": "[trim(parameters('computeNodeNamePrefix'))]",
    "managedIdentity": {
      "type": "SystemAssigned"
    },
    "emptyArray": [],
    "diskTypes": {
      "Standard_HDD": "Standard_LRS",
      "Standard_SSD": "StandardSSD_LRS",
      "Premium_SSD": "Premium_LRS"
    },
    "headNodeDataDisks": {
      "copy": [
        {
          "name": "hnDataDisks",
          "count": "[if(equals(parameters('headNodeDataDiskCount'), 0), 1, parameters('headNodeDataDiskCount'))]",
          "input": {
            "lun": "[copyIndex('hnDataDisks')]",
            "createOption": "Empty",
            "diskSizeGB": "[parameters('headNodeDataDiskSize')]",
            "managedDisk": {
              "storageAccountType": "[variables('diskTypes')[parameters('headNodeDataDiskType')]]"
            }
          }
        }
      ]
    },
    "storageAccountName": "[concat('hpc', uniqueString(resourceGroup().id, variables('clusterName')))]",
    "storageAccountId": "[resourceId('Microsoft.Storage/storageAccounts',variables('storageAccountName'))]",
    "lbName": "[concat(variables('clusterName'),'-lb')]",
    "lbID": "[resourceId('Microsoft.Network/loadBalancers',variables('lbName'))]",
    "lbFrontEndIPConfigID": "[concat(variables('lbID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]",
    "lbPoolID": "[concat(variables('lbID'),'/backendAddressPools/BackendPool1')]",
    "lbProbeID": "[concat(variables('lbID'),'/probes/tcpProbe')]",
    "addressPrefix": "10.0.0.0/16",
    "subnet1Name": "Subnet-1",
    "subnet1Prefix": "10.0.0.0/22",
    "headNodes": "[split(variables('headNodeList'), ',')]",
    "headNodeDscExtNames": [ "setupPrimaryHeadNode", "setupSecondaryHeadNode" ],
    "virtualNetworkName": "[concat(variables('clusterName'),'vnet')]",
    "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]",
    "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnet1Name'))]",
    "privateClusterFQDN": "[concat(toLower(variables('clusterName')), '.', variables('domainName'))]",
    "publicIPSuffix": "[uniqueString(resourceGroup().id)]",
    "publicIPName": "[concat(variables('clusterName'),'publicip')]",
    "publicIPDNSNameLabel": "[concat(toLower(variables('clusterName')), variables('publicIPSuffix'))]",
    "publicIPAddressType": "Dynamic",
    "publicIpAddressId": {
      "id": "[resourceId('Microsoft.Network/publicIPAddresses', variables('publicIPName'))]"
    },
    "availabilitySetNameHN": "[concat(variables('clusterName'), '-avset')]",
    "cnAvailabilitySetName": "[concat(variables('computeNodeNamePrefix'), 'avset')]",
    "nbrVMPerAvailabilitySet": 200,
    "cnAvailabilitySetNumber": "[add(div(parameters('computeNodeNumber'), variables('nbrVMPerAvailabilitySet')), 1)]",
    "uniqueSuffix": "[uniqueString(variables('subnetRef'))]",
    "uniqueNicSuffix": "[concat('-nic-', variables('uniqueSuffix'))]",
    "uniquePubNicSuffix": "[concat('-pubnic-', variables('uniqueSuffix'))]",
    "dcVMName": "[concat(variables('clusterName'),'dc')]",
    "nicNameDC": "[concat(variables('dcVMName'), variables('uniqueNicSuffix'))]",
    "nsgName": "[concat('hpcnsg-',uniquestring(resourceGroup().id, variables('subnetRef')))]",
    "rdmaASeries": [
      "Standard_A8",
      "Standard_A9"
    ],
    "cnRDMACapable": "[or(contains(variables('rdmaASeries'), parameters('computeNodeVMSize')), contains(toLower(split(parameters('computeNodeVMSize'), '_')[1]), 'r'))]",
    "hnRDMACapable": "[or(contains(variables('rdmaASeries'), parameters('headNodeVMSize')), contains(toLower(split(parameters('headNodeVMSize'), '_')[1]), 'r'))]",
    "createCNInAVSet": "[or(equals(parameters('computeNodeInAVSet'), 'Yes'), and(equals(parameters('computeNodeInAVSet'), 'Auto'), variables('cnRDMACapable')))]",
    "useVmssForCN": "[equals(parameters('useVmssForComputeNodes'), 'Yes')]",
    "autoEnableInfiniBand": "[equals(parameters('autoInstallInfiniBandDriver'), 'Yes')]",
    "vmPriority": "[if(equals(parameters('useSpotInstanceForComputeNodes'), 'Yes'), 'Spot', 'Regular')]",
    "computeVmssName": "[take(replace(variables('computeNodeNamePrefix'), '-', ''), 9)]",
    "vmssSinglePlacementGroup": "[lessOrEquals(parameters('computeNodeNumber'), 100)]",
    "certSecrets": [
      {
        "sourceVault": {
          "id": "[resourceId(variables('vaultResourceGroup'), 'Microsoft.KeyVault/vaults', variables('vaultName'))]"
        },
        "vaultCertificates": [
          {
            "certificateUrl": "[parameters('certificateUrl')]",
            "certificateStore": "My"
          }
        ]
      }
    ],
    "headNodeImages": {
      "WindowsServer2016": {
        "publisher": "MicrosoftWindowsServerHPCPack",
        "offer": "WindowsServerHPCPack",
        "sku": "2019hn-ws2016",
        "version": "latest"
      },
      "WindowsServer2019": {
        "publisher": "MicrosoftWindowsServerHPCPack",
        "offer": "WindowsServerHPCPack",
        "sku": "2019hn-ws2019",
        "version": "6.0.7531"
      },
      "CustomImage": {
        "id": "[trim(parameters('headNodeImageResourceId'))]"
      }
    },
    "computeNodeImages": {
      "WindowsServer2008R2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2008-R2-SP1",
        "version": "latest"
      },
      "WindowsServer2012": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2012-Datacenter",
        "version": "latest"
      },
      "WindowsServer2012R2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2012-R2-Datacenter",
        "version": "latest"
      },
      "WindowsServer2016": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2016-Datacenter",
        "version": "latest"
      },
      "WindowsServer2019": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2019-Datacenter",
        "version": "latest"
      },
      "WindowsServer2022": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2022-datacenter",
        "version": "latest"
      },
      "WindowsServer2012R2WithExcel": {
        "publisher": "MicrosoftWindowsServerHPCPack",
        "offer": "WindowsServerHPCPack",
        "sku": "2016U2CN-WS2012R2-Excel",
        "version": "latest"
      },
      "WindowsServer2016WithExcel": {
        "publisher": "MicrosoftWindowsServerHPCPack",
        "offer": "WindowsServerHPCPack",
        "sku": "2016U2CN-WS2016-Excel",
        "version": "latest"
      },
      "WindowsServer2012_Gen2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2012-datacenter-gensecond",
        "version": "latest"
      },
      "WindowsServer2012R2_Gen2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2012-r2-datacenter-gensecond",
        "version": "latest"
      },
      "WindowsServer2016_Gen2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2016-datacenter-gensecond",
        "version": "latest"
      },
      "WindowsServer2019_Gen2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2019-datacenter-gensecond",
        "version": "latest"
      },
      "WindowsServer2022_Gen2": {
        "publisher": "MicrosoftWindowsServer",
        "offer": "WindowsServer",
        "sku": "2022-datacenter-g2",
        "version": "latest"
      },
      "CustomImage": {
        "id": "[trim(parameters('computeNodeImageResourceId'))]"
      }
    },
    "headNodeImageRef": "[variables('headNodeImages')[parameters('headNodeOS')]]",
    "computeNodeImageRef": "[variables('computeNodeImages')[parameters('computeNodeImage')]]",
    "sharedResxBaseUrl": "https://raw.githubusercontent.com/Azure/hpcpack-template/master/HPCPack2019/shared-resources"
  },
  "resources": [
    {
      "type": "Microsoft.Storage/storageAccounts",
      "name": "[variables('storageAccountName')]",
      "apiVersion": "2019-04-01",
      "location": "[resourceGroup().location]",
      "sku": {
        "name": "Standard_LRS"
      },
      "kind": "Storage",
      "properties": {}
    },
    {
      "apiVersion": "2019-04-01",
      "type": "Microsoft.Network/virtualNetworks",
      "name": "[variables('virtualNetworkName')]",
      "location": "[resourceGroup().location]",
      "properties": {
        "addressSpace": {
          "addressPrefixes": [
            "[variables('addressPrefix')]"
          ]
        },
        "subnets": [
          {
            "name": "[variables('subnet1Name')]",
            "properties": {
              "addressPrefix": "[variables('subnet1Prefix')]"
            }
          }
        ]
      }
    },
    {
      "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]",
      "apiVersion": "2019-04-01",
      "type": "Microsoft.Network/publicIPAddresses",
      "name": "[variables('publicIPName')]",
      "location": "[resourceGroup().location]",
      "sku": {
        "name": "Basic"
      },
      "properties": {
        "publicIPAllocationMethod": "[variables('publicIPAddressType')]",
        "dnsSettings": {
          "domainNameLabel": "[variables('publicIPDNSNameLabel')]"
        }
      }
    },
    {
      "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]",
      "type": "Microsoft.Network/networkSecurityGroups",
      "name": "[variables('nsgName')]",
      "apiVersion": "2019-04-01",
      "location": "[resourceGroup().location]",
      "properties": {
        "securityRules": [
          {
            "name": "allow-HTTPS",
            "properties": {
              "description": "Allow Https",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "443",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1000,
              "direction": "Inbound"
            }
          },
          {
            "name": "allow-RDP",
            "properties": {
              "description": "Allow RDP",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "3389",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1010,
              "direction": "Inbound"
            }
          },
          {
            "name": "allow-HPCSession",
            "properties": {
              "description": "Allow HPC Session service",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "9090",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1020,
              "direction": "Inbound"
            }
          },
          {
            "name": "allow-HPCBroker",
            "properties": {
              "description": "Allow HPC Broker service",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "9087",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1030,
              "direction": "Inbound"
            }
          },
          {
            "name": "allow-HPCBrokerWorker",
            "properties": {
              "description": "Allow HPC Broker worker",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "9091",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1040,
              "direction": "Inbound"
            }
          },
          {
            "name": "allow-HPCDataService",
            "properties": {
              "description": "Allow HPC Data service",
              "protocol": "Tcp",
              "sourcePortRange": "*",
              "destinationPortRange": "9094 ",
              "sourceAddressPrefix": "*",
              "destinationAddressPrefix": "*",
              "access": "Allow",
              "priority": 1050,
              "direction": "Inbound"
            }
          }
        ]
      }
    },
    {
      "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]",
      "name": "[concat(variables('headNodes')[copyIndex()], 'PublicIp')]",
      "type": "Microsoft.Network/publicIPAddresses",
      "apiVersion": "2019-04-01",
      "location": "[resourceGroup().location]",
      "copy": {
        "name": "HNPips",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "publicIPAllocationMethod": "Dynamic",
        "dnsSettings": {
          "domainNameLabel": "[toLower(concat(variables('headNodes')[copyIndex()], variables('publicIPSuffix')))]"
        }
      }
    },
    {
      "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]",
      "apiVersion": "2019-04-01",
      "name": "[variables('lbName')]",
      "type": "Microsoft.Network/loadBalancers",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]",
        "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPName'))]"
      ],
      "properties": {
        "frontendIPConfigurations": [
          {
            "name": "LoadBalancerFrontEnd",
            "properties": {
              "publicIPAddress": "[variables('publicIpAddressId')]"
            }
          }
        ],
        "backendAddressPools": [
          {
            "name": "BackendPool1"
          }
        ],
        "inboundNatRules": [
          {
            "name": "[concat('RDP-', variables('headNodes')[0])]",
            "properties": {
              "frontendIPConfiguration": {
                "id": "[variables('lbFrontEndIPConfigID')]"
              },
              "protocol": "Tcp",
              "frontendPort": 50001,
              "backendPort": 3389,
              "enableFloatingIP": false
            }
          },
          {
            "name": "[concat('RDP-', variables('headNodes')[1])]",
            "properties": {
              "frontendIPConfiguration": {
                "id": "[variables('lbFrontEndIPConfigID')]"
              },
              "protocol": "Tcp",
              "frontendPort": 50002,
              "backendPort": 3389,
              "enableFloatingIP": false
            }
          }
        ],
        "loadBalancingRules": [
          {
            "name": "LBRule",
            "properties": {
              "frontendIPConfiguration": {
                "id": "[variables('lbFrontEndIPConfigID')]"
              },
              "backendAddressPool": {
                "id": "[variables('lbPoolID')]"
              },
              "protocol": "Tcp",
              "frontendPort": 443,
              "backendPort": 443,
              "enableFloatingIP": false,
              "idleTimeoutInMinutes": 5,
              "probe": {
                "id": "[variables('lbProbeID')]"
              }
            }
          }
        ],
        "probes": [
          {
            "name": "tcpProbe",
            "properties": {
              "protocol": "Tcp",
              "port": 5802,
              "intervalInSeconds": 5,
              "numberOfProbes": 2
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2019-04-01",
      "type": "Microsoft.Network/networkInterfaces",
      "name": "[variables('nicNameDC')]",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Network/virtualNetworks/', variables('virtualNetworkName'))]"
      ],
      "properties": {
        "ipConfigurations": [
          {
            "name": "IPConfig",
            "properties": {
              "privateIPAllocationMethod": "Static",
              "privateIPAddress": "10.0.0.4",
              "subnet": {
                "id": "[variables('subnetRef')]"
              }
            }
          }
        ]
      }
    },
    {
      "apiVersion": "2019-03-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[variables('dcVMName')]",
      "dependsOn": [
        "[concat('Microsoft.Network/networkInterfaces/', variables('nicNameDC'))]"
      ],
      "location": "[resourceGroup().location]",
      "properties": {
        "hardwareProfile": {
          "vmSize": "[variables('dcSize')]"
        },
        "osProfile": {
          "computerName": "[variables('dcVMName')]",
          "adminUsername": "[parameters('adminUsername')]",
          "adminPassword": "[parameters('adminPassword')]"
        },
        "storageProfile": {
          "imageReference": {
            "publisher": "MicrosoftWindowsServer",
            "offer": "WindowsServer",
            "sku": "2019-Datacenter",
            "version": "latest"
          },
          "osDisk": {
            "name": "[concat(variables('dcVMName'), '-osdisk')]",
            "caching": "ReadOnly",
            "createOption": "FromImage",
            "managedDisk": {
              "storageAccountType": "Standard_LRS"
            }
          },
          "dataDisks": [
            {
              "name": "[concat(variables('dcVMName'), '-datadisk')]",
              "caching": "None",
              "createOption": "Empty",
              "managedDisk": {
                "storageAccountType": "Standard_LRS"
              },
              "diskSizeGB": 200,
              "lun": 0
            }
          ]
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', variables('nicNameDC'))]"
            }
          ]
        }
      }
    },
    {
      "apiVersion": "2019-03-01",
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('dcVMName'), '/promoteDomainController')]",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('dcVMName'))]"
      ],
      "properties": {
        "publisher": "Microsoft.Powershell",
        "type": "DSC",
        "typeHandlerVersion": "2.80",
        "autoUpgradeMinorVersion": true,
        "settings": {
          "configuration": {
            "url": "[concat(variables('sharedResxBaseUrl'), '/CreateADPDC.ps1.zip')]",
            "script": "CreateADPDC.ps1",
            "function": "CreateADPDC"
          },
          "configurationArguments": {
            "DomainName": "[variables('domainName')]"
          }
        },
        "protectedSettings": {
          "configurationArguments": {
            "AdminCreds": {
              "UserName": "[parameters('adminUsername')]",
              "Password": "[parameters('adminPassword')]"
            }
          }
        }
      }
    },
    {
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "getVNetName",
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": [
          ],
          "outputs": {
            "virtualNetworkName": {
              "type": "string",
              "value": "[variables('virtualNetworkName')]"
            }
          }
        }
      }
    },
    {
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "updateVNetDNS",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('dcVMName'), '/extensions/promoteDomainController')]",
        "getVNetName"
      ],
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": [
            {
              "apiVersion": "2019-04-01",
              "name": "[reference('getVNetName').outputs.virtualNetworkName.value]",
              "type": "Microsoft.Network/virtualNetworks",
              "location": "[resourceGroup().location]",
              "properties": {
                "addressSpace": {
                  "addressPrefixes": [
                    "[variables('addressPrefix')]"
                  ]
                },
                "dhcpOptions": {
                  "dnsServers": [
                    "10.0.0.4",
                    "8.8.8.8"
                  ]
                },
                "subnets": [
                  {
                    "name": "[variables('subnet1Name')]",
                    "properties": {
                      "addressPrefix": "[variables('subnet1Prefix')]"
                    }
                  }
                ]
              }
            }
          ]
        }
      }
    },
    {
      "condition": "[equals(parameters('createPublicIPAddressForHeadNode'), 'Yes')]",
      "apiVersion": "2019-04-01",
      "type": "Microsoft.Network/networkInterfaces",
      "name": "[concat(variables('headNodes')[copyIndex()], variables('uniquePubNicSuffix'))]",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Network/networkInterfaces/', variables('nicNameDC'))]",
        "[concat('Microsoft.Network/networkSecurityGroups/', variables('nsgName'))]",
        "[concat('Microsoft.Network/loadBalancers/', variables('lbName'))]",
        "[concat('Microsoft.Network/publicIPAddresses/', concat(variables('headNodes')[copyIndex()], 'PublicIp'))]"
      ],
      "copy": {
        "name": "HNNicsWithPubicIP",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "ipConfigurations": [
          {
            "name": "IPConfig",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "subnet": {
                "id": "[variables('subnetRef')]"
              },
              "loadBalancerBackendAddressPools": [
                {
                  "id": "[variables('lbPoolID')]"
                }
              ],
              "loadBalancerInboundNatRules": [
                {
                  "id": "[concat(variables('lbID'),'/inboundNatRules/RDP-', variables('headNodes')[copyIndex()])]"
                }
              ],
              "publicIPAddress": {
                "id": "[resourceId('Microsoft.Network/publicIPAddresses', concat(variables('headNodes')[copyIndex()], 'PublicIp'))]"
              }
            }
          }
        ],
        "networkSecurityGroup": {
          "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('nsgName'))]"
        },
        "enableAcceleratedNetworking": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]"
      }
    },
    {
      "condition": "[not(equals(parameters('createPublicIPAddressForHeadNode'), 'Yes'))]",
      "apiVersion": "2019-04-01",
      "type": "Microsoft.Network/networkInterfaces",
      "name": "[concat(variables('headNodes')[copyIndex()], variables('uniqueNicSuffix'))]",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Network/networkInterfaces/', variables('nicNameDC'))]"
      ],
      "copy": {
        "name": "HNNicsWithoutPublicIP",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "ipConfigurations": [
          {
            "name": "IPConfig",
            "properties": {
              "privateIPAllocationMethod": "Dynamic",
              "subnet": {
                "id": "[variables('subnetRef')]"
              }
            }
          }
        ],
        "enableAcceleratedNetworking": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]"
      }
    },
    {
      "type": "Microsoft.Compute/availabilitySets",
      "name": "[variables('availabilitySetNameHN')]",
      "apiVersion": "2019-03-01",
      "sku": {
        "name": "Aligned"
      },
      "properties": {
        "platformUpdateDomainCount": 3,
        "platformFaultDomainCount": 2
      },
      "location": "[resourceGroup().location]"
    },
    {
      "apiVersion": "2019-03-01",
      "type": "Microsoft.Compute/virtualMachines",
      "name": "[variables('headNodes')[copyIndex()]]",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Compute/availabilitySets/', variables('availabilitySetNameHN'))]",
        "[concat('Microsoft.Network/networkInterfaces/', variables('headNodes')[copyIndex()], variables('uniqueNicSuffix'))]",
        "[concat('Microsoft.Network/networkInterfaces/', variables('headNodes')[copyIndex()], variables('uniquePubNicSuffix'))]",
        "Microsoft.Resources/deployments/updateVNetDNS"
      ],
      "copy": {
        "name": "createHeadNodeCopy",
        "count": "[length(variables('headNodes'))]"
      },
      "identity": "[if(equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes'), variables('managedIdentity'), json('null'))]",
      "properties": {
        "availabilitySet": {
          "id": "[resourceId('Microsoft.Compute/availabilitySets', variables('availabilitySetNameHN'))]"
        },
        "hardwareProfile": {
          "vmSize": "[parameters('headNodeVMSize')]"
        },
        "osProfile": {
          "computerName": "[variables('headNodes')[copyIndex()]]",
          "adminUsername": "[parameters('adminUsername')]",
          "adminPassword": "[parameters('adminPassword')]",
          "windowsConfiguration": {
            "enableAutomaticUpdates": false
          },
          "secrets": "[variables('certSecrets')]"
        },
        "storageProfile": {
          "imageReference": "[variables('headNodeImageRef')]",
          "osDisk": {
            "name": "[concat(variables('headNodes')[copyIndex()], '-osdisk')]",
            "caching": "ReadOnly",
            "createOption": "FromImage",
            "managedDisk": {
              "storageAccountType": "[variables('diskTypes')[parameters('headNodeOsDiskType')]]"
            }
          },
          "dataDisks": "[if(equals(parameters('headNodeDataDiskCount'), 0), variables('emptyArray'), variables('headNodeDataDisks').hnDataDisks)]"
        },
        "networkProfile": {
          "networkInterfaces": [
            {
              "id": "[resourceId('Microsoft.Network/networkInterfaces', concat(variables('headNodes')[copyIndex()], if(equals(parameters('createPublicIPAddressForHeadNode'), 'Yes'), variables('uniquePubNicSuffix'), variables('uniqueNicSuffix'))))]"
            }
          ]
        }
      }
    },
    {
      "condition": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]",
      "type": "Microsoft.Authorization/roleAssignments",
      "name": "[guid(resourceGroup().id, variables('clusterName'), variables('headNodes')[copyIndex()])]",
      "apiVersion": "2017-09-01",
      "copy": {
        "name": "msiRoleAssignmentCopy",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c",
        "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()]), '2019-03-01', 'Full').identity.principalId]",
        "scope": "[resourceGroup().id]"
      },
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]"
      ]
    },
    {
      "condition": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]",
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "[concat('msiKeyVaultRoleAssignment', copyIndex())]",
      "copy": {
        "name": "msiKeyVaultRoleAssignmentCopy",
        "count": "[length(variables('headNodes'))]"
      },
      "resourceGroup": "[variables('vaultResourceGroup')]",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]"
      ],
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": [
            {
              "type": "Microsoft.KeyVault/vaults/providers/roleAssignments",
              "name": "[concat(variables('vaultName'), '/Microsoft.Authorization/', guid(variables('vaultName'), variables('vnetID'), variables('headNodes')[copyIndex()]))]",
              "apiVersion": "2017-09-01",
              "properties": {
                "roleDefinitionId": "/providers/Microsoft.Authorization/roleDefinitions/f25e0fa2-a7c8-4377-a976-54943a77a395",
                "principalId": "[reference(resourceId('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()]), '2019-03-01', 'Full').identity.principalId]"
              }
            }
          ]
        }
      }
    },
    {
      "condition": "[and(variables('hnRDMACapable'), variables('autoEnableInfiniBand'))]",
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('headNodes')[copyIndex()], '/installInfiniBandDriver')]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()])]"
      ],
      "copy": {
        "name": "installHNInfiniBandDriverCopy",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "publisher": "Microsoft.HpcCompute",
        "type": "InfiniBandDriverWindows",
        "typeHandlerVersion": "1.2",
        "autoUpgradeMinorVersion": true
      }
    },
    {
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "getSqlDscExtName",
      "properties": {
        "mode": "Incremental",
        "template": {
          "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
          "contentVersion": "1.0.0.0",
          "resources": [
          ],
          "outputs": {
            "extensionName": {
              "type": "string",
              "value": "configSQLServer"
            }
          }
        }
      }
    },
    {
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "[concat('createDBServer', variables('sqlServerVMName'))]",
      "dependsOn": [
        "[concat('Microsoft.Compute/availabilitySets/', variables('availabilitySetNameHN'))]",
        "Microsoft.Resources/deployments/updateVNetDNS",
        "Microsoft.Resources/deployments/getSqlDscExtName"
      ],
      "properties": {
        "mode": "Incremental",
        "templateLink": {
          "uri": "[concat(variables('sharedResxBaseUrl'), '/windowsvm-dsc.json')]",
          "contentVersion": "1.0.0.0"
        },
        "parameters": {
          "subnetId": {
            "value": "[variables('subnetRef')]"
          },
          "vmName": {
            "value": "[variables('sqlServerVMName')]"
          },
          "vmSize": {
            "value": "[parameters('sqlServerVMSize')]"
          },
          "osDiskType": {
            "value": "[variables('diskTypes')[parameters('sqlServerDiskType')]]"
          },
          "imageReference": {
            "value": {
              "publisher": "MicrosoftSQLServer",
              "offer": "SQL2016SP2-WS2016",
              "sku": "Standard",
              "version": "13.2.220526"
            }
          },
          "adminUsername": {
            "value": "[parameters('adminUsername')]"
          },
          "adminPassword": {
            "value": "[parameters('adminPassword')]"
          },
          "availabilitySetName": {
            "value": "[variables('availabilitySetNameHN')]"
          },
          "dataDiskCount":{
            "value": 1
          },
          "dataDiskSizeInGB": {
            "value": 200
          },
          "dataDiskType": {
            "value": "[variables('diskTypes')[parameters('sqlServerDiskType')]]"
          },
          "enableAcceleratedNetworking": {
            "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]"
          },
          "dscExtensionName": {
            "value": "[reference('getSqlDscExtName').outputs.extensionName.value]"
          },
          "dscSettings": {
            "value": {
              "configuration": {
                "url": "[concat(variables('sharedResxBaseUrl'),'/ConfigSQLServer.ps1.zip')]",
                "script": "ConfigSQLServer.ps1",
                "function": "ConfigSQLServer"
              }
            }
          },
          "dscProtectedSettings": {
            "value": {}
          }
        }
      }
    },
    {
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('headNodes')[copyIndex()], '/JoinADDomain')]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[copyIndex()], '/extensions/installInfiniBandDriver')]"
      ],
      "copy": {
        "name": "HNJoinDomainCopy",
        "count": "[length(variables('headNodes'))]"
      },
      "properties": {
        "publisher": "Microsoft.Compute",
        "type": "JsonADDomainExtension",
        "typeHandlerVersion": "1.3",
        "autoUpgradeMinorVersion": true,
        "settings": {
          "Name": "[variables('domainName')]",
          "User": "[concat(variables('domainName'), '\\', parameters('adminUsername'))]",
          "NumberOfRetries": "50",
          "RetryIntervalInMilliseconds": "10000",
          "Restart": "true",
          "Options": "3"
        },
        "protectedSettings": {
          "Password": "[parameters('adminPassword')]"
        }
      }
    },
    {
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('sqlServerVMName'), '/JoinADDomain')]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Resources/deployments/createDBServer', variables('sqlServerVMName'))]"
      ],
      "properties": {
        "publisher": "Microsoft.Compute",
        "type": "JsonADDomainExtension",
        "typeHandlerVersion": "1.3",
        "autoUpgradeMinorVersion": true,
        "settings": {
          "Name": "[variables('domainName')]",
          "User": "[concat(variables('domainName'), '\\', parameters('adminUsername'))]",
          "NumberOfRetries": "50",
          "RetryIntervalInMilliseconds": "10000",
          "Restart": "true",
          "Options": "3"
        },
        "protectedSettings": {
          "Password": "[parameters('adminPassword')]"
        }
      }
    },
    {
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "configDBPermissions",
      "dependsOn": [
        "HNJoinDomainCopy",
        "[concat('Microsoft.Compute/virtualMachines/', variables('sqlServerVMName'), '/extensions/JoinADDomain')]"
      ],
      "properties": {
        "mode": "Incremental",
        "templateLink": {
          "uri": "[concat(variables('sharedResxBaseUrl'),'/dsc-extension.json')]",
          "contentVersion": "1.0.0.0"
        },
        "parameters": {
          "vmName": {
            "value": "[variables('sqlServerVMName')]"
          },
          "dscExtensionName": {
            "value": "[reference('getSqlDscExtName').outputs.extensionName.value]"
          },
          "dscSettings": {
            "value": {
              "configuration": {
                "url": "[concat(variables('sharedResxBaseUrl'),'/ConfigDBPermissions.ps1.zip')]",
                "script": "ConfigDBPermissions.ps1",
                "function": "ConfigDBPermissions"
              },
              "configurationArguments": {
                "DomainName": "[variables('domainName')]",
                "HeadNodeList": "[variables('headNodeList')]"
              }
            }
          },
          "dscProtectedSettings": {
            "value": {
              "configurationArguments": {
                "AdminCreds": {
                  "UserName": "[parameters('adminUsername')]",
                  "Password": "[parameters('adminPassword')]"
                }
              }
            }
          }
        }
      }
    },
    {
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('headNodes')[0],'/setupPrimaryHeadNode')]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "Microsoft.Resources/deployments/configDBPermissions",
        "[concat('Microsoft.Storage/storageAccounts/', variables('storageAccountName'))]"
      ],
      "properties": {
        "publisher": "Microsoft.Powershell",
        "type": "DSC",
        "typeHandlerVersion": "2.80",
        "autoUpgradeMinorVersion": true,
        "settings": {
          "configuration": {
            "url": "[concat(variables('sharedResxBaseUrl'),'/InstallPrimaryHeadNode.ps1.zip')]",
            "script": "InstallPrimaryHeadNode.ps1",
            "function": "InstallPrimaryHeadNode"
          },
          "configurationArguments": {
            "SSLThumbprint": "[variables('certThumbprint')]",
            "ClusterName": "[variables('clusterName')]",
            "SQLServerInstance": "[variables('sqlServerVMName')]",
            "EnableBuiltinHA": true,
            "CNSize": "[parameters('computeNodeVMSize')]",
            "SubscriptionId": "[subscription().subscriptionId]",
            "VNet": "[variables('virtualNetworkName')]",
            "Subnet": "[variables('subnet1Name')]",
            "Location": "[resourceGroup().location]",
            "ResourceGroup": "[resourceGroup().name]",
            "VaultResourceGroup": "[variables('vaultResourceGroup')]",
            "CertificateUrl": "[parameters('CertificateUrl')]",
            "CNNamePrefix": "[variables('computeNodeNamePrefix')]",
            "AutoGSUseManagedIdentity": "[equals(parameters('enableManagedIdentityOnHeadNode'), 'Yes')]"
          }
        },
        "protectedSettings": {
          "configurationArguments": {
            "SetupUserCredential": {
              "UserName": "[concat(variables('domainName'), '\\', parameters('adminUsername'))]",
              "Password": "[parameters('adminPassword')]"
            },
            "AzureStorageConnString": "[concat('DefaultEndpointsProtocol=https;AccountName=', variables('storageAccountName'), ';AccountKey=', listKeys(variables('storageAccountId'), '2019-04-01').keys[0].value)]"
          }
        }
      }
    },
    {
      "type": "Microsoft.Compute/virtualMachines/extensions",
      "name": "[concat(variables('headNodes')[1],'/setupSecondaryHeadNode')]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "dependsOn": [
        "[concat('Microsoft.Compute/virtualMachines/', variables('headNodes')[0], '/extensions/setupPrimaryHeadNode')]"
      ],
      "properties": {
        "publisher": "Microsoft.Powershell",
        "type": "DSC",
        "typeHandlerVersion": "2.80",
        "autoUpgradeMinorVersion": true,
        "settings": {
          "configuration": {
            "url": "[concat(variables('sharedResxBaseUrl'), '/InstallHpcNode.ps1.zip')]",
            "script": "InstallHpcNode.ps1",
            "function": "InstallHpcNode"
          },
          "configurationArguments": {
            "NodeType": "PassiveHeadNode",
            "HeadNodeList": "[variables('headNodeList')]",
            "SSLThumbprint": "[variables('certThumbprint')]"
          }
        },
        "protectedSettings": {
          "configurationArguments": {
            "SetupUserCredential": {
              "UserName": "[concat(variables('domainName'), '\\', parameters('adminUsername'))]",
              "Password": "[parameters('adminPassword')]"
            }
          }
        }
      }
    },
    {
      "condition": "[variables('createCNInAVSet')]",
      "type": "Microsoft.Compute/availabilitySets",
      "name": "[concat(variables('cnAvailabilitySetName'), padLeft(string(copyIndex()), 2, '0'))]",
      "apiVersion": "2019-03-01",
      "location": "[resourceGroup().location]",
      "sku": {
        "name": "Aligned"
      },
      "properties": {
        "platformUpdateDomainCount": 5,
        "platformFaultDomainCount": 2
      },
      "copy": {
        "name": "cnAvailabilitySetCopy",
        "count": "[variables('cnAvailabilitySetNumber')]"
      },
      "dependsOn": [
        "Microsoft.Resources/deployments/updateVNetDNS"
      ]
    },
    {
      "condition": "[and(greater(parameters('computeNodeNumber'), 0), not(variables('useVmssForCN')))]",
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "[concat('create', variables('computeNodeNamePrefix'), padLeft(string(copyIndex()), 3, '0'))]",
      "dependsOn": [
        "Microsoft.Resources/deployments/updateVNetDNS",
        "cnAvailabilitySetCopy"
      ],
      "copy": {
        "name": "CN",
        "count": "[parameters('computeNodeNumber')]"
      },
      "properties": {
        "mode": "Incremental",
        "templateLink": {
          "uri": "[concat(variables('sharedResxBaseUrl'), '/computenode.json')]",
          "contentVersion": "1.0.0.0"
        },
        "parameters": {
          "subnetId": {
            "value": "[variables('subnetRef')]"
          },
          "vmName": {
            "value": "[concat(variables('computeNodeNamePrefix'), padLeft(string(copyIndex()), 3, '0'))]"
          },
          "vmSize": {
            "value": "[parameters('computeNodeVMSize')]"
          },
          "osDiskType": {
            "value": "[variables('diskTypes')[parameters('computeNodeOsDiskType')]]"
          },
          "dataDiskSizeInGB": {
            "value": "[parameters('computeNodeDataDiskSize')]"
          },
          "dataDiskCount": {
            "value": "[parameters('computeNodeDataDiskCount')]"
          },
          "dataDiskType": {
            "value": "[variables('diskTypes')[parameters('computeNodeDataDiskType')]]"
          },
          "imageReference": {
            "value": "[variables('computeNodeImageRef')]"
          },
          "imageOsPlatform": {
            "value": "windows"
          },
          "adminUsername": {
            "value": "[parameters('adminUsername')]"
          },
          "adminPassword": {
            "value": "[parameters('adminPassword')]"
          },
          "availabilitySetName": {
            "value": "[if(variables('createCNInAVSet'), concat(variables('cnAvailabilitySetName'), padLeft(string(div(copyIndex(), variables('nbrVMPerAvailabilitySet'))), 2, '0')), '')]"
          },
          "vmPriority": {
            "value": "[variables('vmPriority')]"
          },
          "installRDMADriver": {
            "value": "[and(variables('cnRDMACapable'), variables('autoEnableInfiniBand'))]"
          },
          "enableAcceleratedNetworking": {
            "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]"
          },
          "secrets": {
            "value": "[variables('certSecrets')]"
          },
          "certThumbprint": {
            "value": "[variables('certThumbprint')]"
          },
          "headNodeList": {
            "value": "[variables('headNodeList')]"
          },
          "joinDomain": {
            "value": true
          },
          "domainName": {
            "value": "[variables('domainName')]"
          }
        }
      }
    },
    {
      "condition": "[and(greater(parameters('computeNodeNumber'), 0), variables('useVmssForCN'))]",
      "apiVersion": "2019-05-01",
      "type": "Microsoft.Resources/deployments",
      "name": "[concat('create', variables('computeVmssName'))]",
      "dependsOn": [
        "Microsoft.Resources/deployments/updateVNetDNS"
      ],
      "properties": {
        "mode": "Incremental",
        "templateLink": {
          "uri": "[concat(variables('sharedResxBaseUrl'), '/computevmss.json')]",
          "contentVersion": "1.0.0.0"
        },
        "parameters": {
          "subnetId": {
            "value": "[variables('subnetRef')]"
          },
          "vmssName": {
            "value": "[variables('computeVmssName')]"
          },
          "vmNumber": {
            "value": "[parameters('computeNodeNumber')]"
          },
          "vmSize": {
            "value": "[parameters('computeNodeVMSize')]"
          },
          "osDiskType": {
            "value": "[variables('diskTypes')[parameters('computeNodeOsDiskType')]]"
          },
          "dataDiskSizeInGB": {
            "value": "[parameters('computeNodeDataDiskSize')]"
          },
          "dataDiskCount": {
            "value": "[parameters('computeNodeDataDiskCount')]"
          },
          "dataDiskType": {
            "value": "[variables('diskTypes')[parameters('computeNodeDataDiskType')]]"
          },
          "imageReference": {
            "value": "[variables('computeNodeImageRef')]"
          },
          "imageOsPlatform": {
            "value": "windows"
          },
          "adminUsername": {
            "value": "[parameters('adminUsername')]"
          },
          "adminPassword": {
            "value": "[parameters('adminPassword')]"
          },
          "singlePlacementGroup": {
            "value": "[variables('vmssSinglePlacementGroup')]"
          },
          "vmPriority": {
            "value": "[variables('vmPriority')]"
          },
          "installRDMADriver": {
            "value": "[and(variables('cnRDMACapable'), variables('autoEnableInfiniBand'))]"
          },
          "enableAcceleratedNetworking": {
            "value": "[equals(parameters('enableAcceleratedNetworking'), 'Yes')]"
          },
          "secrets": {
            "value": "[variables('certSecrets')]"
          },
          "certThumbprint": {
            "value": "[variables('certThumbprint')]"
          },
          "headNodeList": {
            "value": "[variables('headNodeList')]"
          },
          "joinDomain": {
            "value": true
          },
          "domainName": {
            "value": "[variables('domainName')]"
          }
        }
      }
    }
  ],
  "outputs": {
    "clusterDNSName": {
      "type": "string",
      "value": "[if(equals(parameters('createPublicIPAddressForHeadNode'), 'No'), variables('privateClusterFQDN'), reference(variables('publicIpAddressId').id, '2019-04-01').dnsSettings.fqdn)]"
    }
  }
}