зеркало из https://github.com/Azure/az-hop.git
1169 строки
53 KiB
Bicep
1169 строки
53 KiB
Bicep
targetScope = 'resourceGroup'
|
|
|
|
var azhopResourceGroupName = resourceGroup().name
|
|
|
|
@description('Azure region to use')
|
|
param location string
|
|
|
|
@description('Branch name to deploy from - Default main')
|
|
param branchName string = 'main'
|
|
|
|
@description('Autogenerate passwords and SSH key pair.')
|
|
param autogenerateSecrets bool = false
|
|
|
|
@description('SSH Public Key for the Virtual Machines.')
|
|
@secure()
|
|
param adminSshPublicKey string = ''
|
|
|
|
@description('SSH Private Key for the Virtual Machines.')
|
|
@secure()
|
|
param adminSshPrivateKey string = ''
|
|
|
|
@description('The Windows/Active Directory password.')
|
|
@secure()
|
|
param adminPassword string = ''
|
|
|
|
@description('Password for the Slurm accounting admin user')
|
|
@secure()
|
|
param databaseAdminPassword string = ''
|
|
|
|
@description('Identity of the deployer if not deploying from a deployer VM')
|
|
param loggedUserObjectId string = ''
|
|
|
|
@description('Input configuration file in json format')
|
|
param azhopConfig object
|
|
|
|
var resourcePostfix = '${uniqueString(subscription().subscriptionId, azhopResourceGroupName)}x'
|
|
|
|
// Local variables to help in the simplication as functions doesn't exists
|
|
var enablePublicIP = contains(azhopConfig, 'locked_down_network') ? azhopConfig.locked_down_network.public_ip : true
|
|
var jumpboxSshPort = deployJumpbox ? (contains(azhopConfig.jumpbox, 'ssh_port') ? azhopConfig.jumpbox.ssh_port : 22) : 22
|
|
var deployerSshPort = deployDeployer ? (contains(azhopConfig.deployer, 'ssh_port') ? azhopConfig.deployer.ssh_port : 22) : 22
|
|
var ccportalSshPort = cycleCloudAsDeployer ? (contains(azhopConfig.cyclecloud, 'ssh_port') ? azhopConfig.cyclecloud.ssh_port : 22) : 22
|
|
var incomingSSHPort = deployDeployer ? deployerSshPort : (cycleCloudAsDeployer ? ccportalSshPort : jumpboxSshPort )
|
|
|
|
|
|
var deployLustre = contains(azhopConfig, 'lustre') && contains(azhopConfig.lustre, 'create') ? azhopConfig.lustre.create : false
|
|
var deployJumpbox = contains(azhopConfig, 'jumpbox') ? true : false
|
|
var deployDeployer = contains(azhopConfig, 'deployer') ? true : false
|
|
var deployGrafana = contains(azhopConfig, 'monitoring') && contains(azhopConfig.monitoring, 'grafana') ? azhopConfig.monitoring.grafana : true
|
|
var deployOnDemand = contains(azhopConfig, 'ondemand') ? true : false
|
|
var cycleCloudAsDeployer = contains(azhopConfig, 'cyclecloud') && contains(azhopConfig.cyclecloud, 'use_as_deployer') ? azhopConfig.cyclecloud.use_as_deployer : false
|
|
|
|
var useExistingAD = contains(azhopConfig, 'domain') ? azhopConfig.domain.use_existing_dc : false
|
|
var userAuth = contains(azhopConfig, 'authentication') && contains(azhopConfig.authentication, 'user_auth') ? azhopConfig.authentication.user_auth : 'ad'
|
|
var createAD = ! useExistingAD && (userAuth == 'ad')
|
|
|
|
var highAvailabilityForAD = contains(azhopConfig, 'ad') && contains(azhopConfig.ad, 'high_availability') ? azhopConfig.ad.high_availability : false
|
|
|
|
var linuxBaseImage = contains(azhopConfig, 'linux_base_image') ? azhopConfig.linux_base_image : 'OpenLogic:CentOS:7_9-gen2:latest'
|
|
var linuxBasePlan = contains(azhopConfig, 'linux_base_plan') ? azhopConfig.linux_base_plan : ''
|
|
var windowsBaseImage = contains(azhopConfig, 'windows_base_image') ? azhopConfig.windows_base_image : 'MicrosoftWindowsServer:WindowsServer:2019-Datacenter-smalldisk:latest'
|
|
var cyclecloudBaseImage = contains(azhopConfig.cyclecloud, 'image') ? azhopConfig.cyclecloud.image : linuxBaseImage
|
|
var cyclecloudBasePlan = contains(azhopConfig.cyclecloud, 'plan') ? azhopConfig.cyclecloud.plan : linuxBasePlan
|
|
|
|
var createDatabase = queue_manager == 'slurm' && slurm_accounting_enabled
|
|
var slurm_accounting_enabled = contains(azhopConfig, 'slurm') && contains(azhopConfig.slurm, 'accounting_enabled') ? azhopConfig.slurm.accounting_enabled : false
|
|
|
|
var computeMIoption = contains(azhopConfig, 'compute_vm_identity')
|
|
var createComputeMI = computeMIoption && contains(azhopConfig.compute_vm_identity, 'create') ? azhopConfig.compute_vm_identity.create : false
|
|
var computeMIname = computeMIoption && contains(azhopConfig.compute_vm_identity, 'name') ? azhopConfig.compute_vm_identity.name : 'compute-mi'
|
|
var existingComputeMIrg = !createComputeMI && computeMIoption && contains(azhopConfig.compute_vm_identity, 'resource_group') ? azhopConfig.compute_vm_identity.resource_group : ''
|
|
|
|
var create_anf = contains(azhopConfig, 'anf') && contains(azhopConfig.anf, 'create') ? azhopConfig.anf.create : (contains(azhopConfig, 'anf') ? true : false)
|
|
|
|
var nsgTargetForDC = {
|
|
type: useExistingAD ? 'ips' : 'asg'
|
|
target: useExistingAD ? azhopConfig.domain.existing_dc_details.domain_controller_ip_addresses : 'asg-ad'
|
|
}
|
|
|
|
var vmNamesMap = {
|
|
ad : contains(azhopConfig, 'ad') && contains(azhopConfig.ad, 'name') ? azhopConfig.ad.name : 'ad'
|
|
ad2 : contains(azhopConfig, 'ad') && contains(azhopConfig.ad, 'ha_name') ? azhopConfig.ad.ha_name : 'ad2'
|
|
deployer: contains(azhopConfig, 'deployer') && contains(azhopConfig.deployer, 'name') ? azhopConfig.deployer.name : 'deployer'
|
|
jumpbox: contains(azhopConfig, 'jumpbox') && contains(azhopConfig.jumpbox, 'name') ? azhopConfig.jumpbox.name : 'jumpbox'
|
|
ccportal: contains(azhopConfig, 'cyclecloud') && contains(azhopConfig.cyclecloud, 'name') ? azhopConfig.cyclecloud.name : 'ccportal'
|
|
ondemand: contains(azhopConfig, 'ondemand') && contains(azhopConfig.ondemand, 'name') ? azhopConfig.ondemand.name : 'ondemand'
|
|
scheduler: contains(azhopConfig, 'scheduler') && contains(azhopConfig.scheduler, 'name') ? azhopConfig.scheduler.name : 'scheduler'
|
|
grafana: contains(azhopConfig, 'grafana') && contains(azhopConfig.grafana, 'name') ? azhopConfig.grafana.name : 'grafana'
|
|
}
|
|
var queue_manager= contains(azhopConfig, 'queue_manager') ? azhopConfig.queue_manager : 'openpbs'
|
|
// Convert the azhop configuration file to a pivot format used for the deployment
|
|
var config = {
|
|
admin_user: azhopConfig.admin_user
|
|
keyvault_readers: contains(azhopConfig, 'key_vault_readers') ? ( empty(azhopConfig.key_vault_readers) ? [] : [ azhopConfig.key_vault_readers ] ) : []
|
|
|
|
public_ip: enablePublicIP
|
|
deploy_gateway: contains(azhopConfig, 'vpn_gateway') && contains(azhopConfig.vpn_gateway, 'create') ? azhopConfig.vpn_gateway.create : false
|
|
deploy_bastion: contains(azhopConfig, 'bastion') && contains(azhopConfig.bastion, 'create') ? azhopConfig.bastion.create : false
|
|
deploy_lustre: deployLustre
|
|
|
|
|
|
lock_down_network: {
|
|
enforce: contains(azhopConfig, 'locked_down_network') && contains(azhopConfig.locked_down_network, 'enforce') ? azhopConfig.locked_down_network.enforce : false
|
|
grant_access_from: contains(azhopConfig, 'locked_down_network') && contains(azhopConfig.locked_down_network, 'grant_access_from') ? ( empty(azhopConfig.locked_down_network.grant_access_from) ? [] : [ azhopConfig.locked_down_network.grant_access_from ] ) : []
|
|
}
|
|
|
|
nat_gateway: {
|
|
create: contains(azhopConfig, 'nat_gateway') && contains(azhopConfig.nat_gateway, 'create') ? azhopConfig.nat_gateway.create : false
|
|
name: contains(azhopConfig, 'nat_gateway') && contains(azhopConfig.nat_gateway, 'name') ? azhopConfig.nat_gateway.name : 'natgw-${resourcePostfix}'
|
|
}
|
|
|
|
queue_manager: queue_manager
|
|
|
|
slurm: {
|
|
admin_user: contains(azhopConfig, 'database') && contains(azhopConfig.database, 'user') ? azhopConfig.database.user : 'sqladmin'
|
|
accounting_enabled: contains(azhopConfig, 'slurm') && contains(azhopConfig.slurm, 'accounting_enabled') ? azhopConfig.slurm.accounting_enabled : false
|
|
}
|
|
|
|
private_dns: {
|
|
create: contains(azhopConfig, 'private_dns') && contains(azhopConfig.private_dns, 'create') ? azhopConfig.private_dns.create : false
|
|
name: contains(azhopConfig, 'private_dns') && contains(azhopConfig.private_dns, 'name') ? azhopConfig.private_dns.name : 'hpc.azure'
|
|
registration_enabled: contains(azhopConfig, 'private_dns') && contains(azhopConfig.private_dns, 'registration_enabled') ? azhopConfig.private_dns.registration_enabled : false
|
|
}
|
|
|
|
|
|
domain: {
|
|
name : contains(azhopConfig, 'domain') ? azhopConfig.domain.name : 'hpc.azure'
|
|
domain_join_user: createAD ? {
|
|
username: azhopConfig.admin_user
|
|
password_key_vault_name: 'foo'
|
|
password_key_vault_resource_group_name: 'foo'
|
|
password_key_vault_secret_name: 'foo'
|
|
} : useExistingAD ? {
|
|
username: azhopConfig.domain.domain_join_user.username
|
|
password_key_vault_name: azhopConfig.domain.domain_join_user.password_key_vault_name
|
|
password_key_vault_resource_group_name: azhopConfig.domain.domain_join_user.password_key_vault_resource_group_name
|
|
password_key_vault_secret_name: azhopConfig.domain.domain_join_user.password_key_vault_secret_name
|
|
} : {
|
|
username: ''
|
|
}
|
|
domain_controlers : createAD ? (! highAvailabilityForAD ? [vmNamesMap.ad] : [vmNamesMap.ad, vmNamesMap.ad2]) : useExistingAD ? azhopConfig.domain.existing_dc_details.domain_controller_names : []
|
|
ldap_server: createAD ? vmNamesMap.ad : useExistingAD ? azhopConfig.domain.existing_dc_details.domain_controller_names[0] : ''
|
|
}
|
|
|
|
key_vault_name: contains(azhopConfig, 'azure_key_vault') ? azhopConfig.azure_key_vault.name : 'kv${resourcePostfix}'
|
|
storage_account_name: contains(azhopConfig, 'azure_storage_account') ? azhopConfig.azure_storage_account.name : 'azhop${resourcePostfix}'
|
|
db_name: contains(azhopConfig, 'database') && contains(azhopConfig.database, 'name') ? azhopConfig.database.name : 'mysql-${resourcePostfix}'
|
|
|
|
deploy_grafana: deployGrafana
|
|
deploy_ondemand: deployOnDemand
|
|
deploy_sig: contains(azhopConfig, 'image_gallery') && contains(azhopConfig.image_gallery, 'create') ? azhopConfig.image_gallery.create : false
|
|
|
|
// Default home directory is ANF
|
|
homedir_type: contains(azhopConfig.mounts.home, 'type') ? azhopConfig.mounts.home.type : 'existing'
|
|
homedir_mountpoint: azhopConfig.mounts.home.mountpoint
|
|
|
|
lustre: {
|
|
create: deployLustre ? true : false
|
|
sku: contains(azhopConfig, 'lustre') && contains(azhopConfig.lustre, 'sku') ? azhopConfig.lustre.sku : 'AMLFS-Durable-Premium-250'
|
|
capacity: contains(azhopConfig, 'lustre') && contains(azhopConfig.lustre, 'capacity') ? azhopConfig.lustre.capacity : 8
|
|
}
|
|
|
|
anf: {
|
|
create: create_anf
|
|
dual_protocol: contains(azhopConfig, 'anf') && contains(azhopConfig.anf, 'dual_protocol') ? azhopConfig.anf.dual_protocol : false
|
|
service_level: contains(azhopConfig, 'anf') && contains(azhopConfig.anf, 'homefs_service_level') ? azhopConfig.anf.homefs_service_level : 'Standard'
|
|
size_gb: contains(azhopConfig, 'anf') && contains(azhopConfig.anf, 'homefs_size_tb') ? azhopConfig.anf.homefs_size_tb*1024 : 4096
|
|
}
|
|
|
|
azurefiles: {
|
|
create: contains(azhopConfig, 'azurefiles') && contains(azhopConfig.azurefiles, 'create') ? azhopConfig.azurefiles.create : false
|
|
size_gb: contains(azhopConfig, 'azurefiles') && contains(azhopConfig.azurefiles, 'size_gb') ? azhopConfig.azurefiles.size_gb : 1024
|
|
}
|
|
|
|
vnet: {
|
|
tags: contains(azhopConfig.network.vnet,'tags') ? azhopConfig.network.vnet.tags : {}
|
|
name: azhopConfig.network.vnet.name
|
|
cidr: azhopConfig.network.vnet.address_space
|
|
subnets: union (
|
|
{
|
|
frontend: {
|
|
name: contains(azhopConfig.network.vnet.subnets.frontend, 'name') ? azhopConfig.network.vnet.subnets.frontend.name : 'frontend'
|
|
cidr: azhopConfig.network.vnet.subnets.frontend.address_prefixes
|
|
nat_gateway: true
|
|
service_endpoints: [
|
|
'Microsoft.Storage'
|
|
]
|
|
}
|
|
admin: {
|
|
name: contains(azhopConfig.network.vnet.subnets.admin, 'name') ? azhopConfig.network.vnet.subnets.admin.name : 'admin'
|
|
cidr: azhopConfig.network.vnet.subnets.admin.address_prefixes
|
|
nat_gateway: true
|
|
service_endpoints: [
|
|
'Microsoft.KeyVault'
|
|
'Microsoft.Storage'
|
|
]
|
|
}
|
|
compute: {
|
|
name: contains(azhopConfig.network.vnet.subnets.compute, 'name') ? azhopConfig.network.vnet.subnets.compute.name : 'compute'
|
|
cidr: azhopConfig.network.vnet.subnets.compute.address_prefixes
|
|
nat_gateway: true
|
|
service_endpoints: [
|
|
'Microsoft.Storage'
|
|
]
|
|
}
|
|
},
|
|
createDatabase ? {
|
|
database: {
|
|
name: contains(azhopConfig.network.vnet.subnets.database, 'name') ? azhopConfig.network.vnet.subnets.database.name : 'database'
|
|
cidr: azhopConfig.network.vnet.subnets.database.address_prefixes
|
|
delegations: [
|
|
'Microsoft.DBforMySQL/flexibleServers'
|
|
]
|
|
}
|
|
} : {},
|
|
create_anf ? {
|
|
netapp: {
|
|
name: contains(azhopConfig.network.vnet.subnets.netapp, 'name') ? azhopConfig.network.vnet.subnets.netapp.name : 'netapp'
|
|
cidr: azhopConfig.network.vnet.subnets.netapp.address_prefixes
|
|
delegations: [
|
|
'Microsoft.Netapp/volumes'
|
|
]
|
|
}
|
|
} : {},
|
|
deployLustre ? {
|
|
lustre: {
|
|
name: contains(azhopConfig.network.vnet.subnets.lustre, 'name') ? azhopConfig.network.vnet.subnets.lustre.name : 'lustre'
|
|
cidr: azhopConfig.network.vnet.subnets.lustre.address_prefixes
|
|
}
|
|
} : {},
|
|
createAD ? {
|
|
ad: {
|
|
name: contains(azhopConfig.network.vnet.subnets.ad, 'name') ? azhopConfig.network.vnet.subnets.ad.name : 'ad'
|
|
cidr: azhopConfig.network.vnet.subnets.ad.address_prefixes
|
|
nat_gateway: true
|
|
}
|
|
} : {},
|
|
contains(azhopConfig.network.vnet.subnets,'bastion') ? {
|
|
bastion: {
|
|
apply_nsg: true
|
|
name: 'AzureBastionSubnet'
|
|
cidr: azhopConfig.network.vnet.subnets.bastion.address_prefixes
|
|
}
|
|
} : {},
|
|
contains(azhopConfig.network.vnet.subnets,'outbounddns') ? {
|
|
outbounddns: {
|
|
name: contains(azhopConfig.network.vnet.subnets.outbounddns, 'name') ? azhopConfig.network.vnet.subnets.outbounddns.name : 'outbounddns'
|
|
cidr: azhopConfig.network.vnet.subnets.outbounddns.address_prefixes
|
|
delegations: [
|
|
'Microsoft.Network/dnsResolvers'
|
|
]
|
|
}
|
|
} : {},
|
|
contains(azhopConfig.network.vnet.subnets,'gateway') ? {
|
|
gateway: {
|
|
apply_nsg: false
|
|
name: 'GatewaySubnet'
|
|
cidr: azhopConfig.network.vnet.subnets.gateway.address_prefixes
|
|
}
|
|
} : {}
|
|
)
|
|
peerings: contains(azhopConfig.network,'peering') ? azhopConfig.network.peering : []
|
|
}
|
|
|
|
images: {
|
|
ubuntu: {
|
|
ref: {
|
|
publisher: 'Canonical'
|
|
offer: '0001-com-ubuntu-server-focal'
|
|
sku: '20_04-lts-gen2'
|
|
version: 'latest'
|
|
}
|
|
}
|
|
linux_base: {
|
|
plan: linuxBasePlan
|
|
ref: contains(linuxBaseImage, '/') ? {
|
|
id: linuxBaseImage
|
|
} : {
|
|
publisher: split(linuxBaseImage,':')[0]
|
|
offer: split(linuxBaseImage,':')[1]
|
|
sku: split(linuxBaseImage,':')[2]
|
|
version: split(linuxBaseImage,':')[3]
|
|
}
|
|
}
|
|
win_base: {
|
|
ref: contains(windowsBaseImage, '/') ? {
|
|
id: windowsBaseImage
|
|
} : {
|
|
publisher: split(windowsBaseImage,':')[0]
|
|
offer: split(windowsBaseImage,':')[1]
|
|
sku: split(windowsBaseImage,':')[2]
|
|
version: split(windowsBaseImage,':')[3]
|
|
}
|
|
}
|
|
cyclecloud_base: {
|
|
plan: cyclecloudBasePlan
|
|
ref: contains(cyclecloudBaseImage, '/') ? {
|
|
id: cyclecloudBaseImage
|
|
} : {
|
|
publisher: split(cyclecloudBaseImage,':')[0]
|
|
offer: split(cyclecloudBaseImage,':')[1]
|
|
sku: split(cyclecloudBaseImage,':')[2]
|
|
version: split(cyclecloudBaseImage,':')[3]
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
vms: union(
|
|
deployOnDemand ? {
|
|
ondemand: {
|
|
subnet: 'frontend'
|
|
name: vmNamesMap.ondemand
|
|
sku: azhopConfig.ondemand.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'linux_base'
|
|
pip: enablePublicIP
|
|
asgs: union(
|
|
[ 'asg-ssh', 'asg-ondemand', 'asg-nfs-client', 'asg-sched', 'asg-cyclecloud-client' ],
|
|
deployGrafana ? ['asg-telegraf'] : [],
|
|
(userAuth == 'ad') ? ['asg-ad-client'] : [],
|
|
deployLustre ? [ 'asg-lustre-client' ] : []
|
|
)
|
|
}
|
|
} : {},
|
|
{
|
|
ccportal: {
|
|
subnet: deployOnDemand ? 'admin' : 'frontend'
|
|
name: vmNamesMap.ccportal
|
|
sku: azhopConfig.cyclecloud.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'cyclecloud_base'
|
|
pip: enablePublicIP && !deployOnDemand
|
|
sshPort: cycleCloudAsDeployer ? incomingSSHPort : 22
|
|
deploy_script: cycleCloudAsDeployer ? replace(replace(loadTextContent('install.sh'), '__INSERT_AZHOP_BRANCH__', branchName), '__SSH_PORT__', string(incomingSSHPort)) : ''
|
|
datadisks: [
|
|
{
|
|
name: '${vmNamesMap.ccportal}-datadisk0'
|
|
disksku: 'Premium_LRS'
|
|
size: split(cyclecloudBaseImage,':')[0] == 'azurecyclecloud' ? 0 : 128
|
|
caching: 'ReadWrite'
|
|
createOption: split(cyclecloudBaseImage,':')[0] == 'azurecyclecloud' ? 'FromImage' : 'Empty'
|
|
}
|
|
]
|
|
identity: {
|
|
keyvault: cycleCloudAsDeployer ? {
|
|
secret_permissions: [ 'All' ]
|
|
} : {}
|
|
roles: [
|
|
'Contributor'
|
|
]
|
|
}
|
|
asgs: union(
|
|
[ 'asg-ssh', 'asg-cyclecloud' ],
|
|
cycleCloudAsDeployer ? [ 'asg-jumpbox', 'asg-deployer' ] : [],
|
|
(userAuth == 'ad') ? ['asg-ad-client'] : [],
|
|
deployGrafana ? ['asg-telegraf'] : []
|
|
)
|
|
}
|
|
scheduler: {
|
|
subnet: 'admin'
|
|
name: vmNamesMap.scheduler
|
|
sku: azhopConfig.scheduler.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'linux_base'
|
|
asgs: union(
|
|
[ 'asg-ssh', 'asg-sched', 'asg-cyclecloud-client', 'asg-nfs-client' ],
|
|
(userAuth == 'ad') ? ['asg-ad-client'] : [],
|
|
deployGrafana ? ['asg-telegraf'] : [],
|
|
createDatabase ? ['asg-mysql-client'] : []
|
|
)
|
|
}
|
|
},
|
|
createAD ? {
|
|
ad: {
|
|
subnet: 'ad'
|
|
windows: true
|
|
name: vmNamesMap.ad
|
|
ahub: contains(azhopConfig.ad, 'hybrid_benefit') ? azhopConfig.ad.hybrid_benefit : false
|
|
sku: azhopConfig.ad.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'win_base'
|
|
asgs: [ 'asg-ad', 'asg-rdp', 'asg-ad-client' ]
|
|
}
|
|
} : {},
|
|
deployDeployer ? {
|
|
deployer: {
|
|
subnet: 'frontend'
|
|
name: vmNamesMap.deployer
|
|
sku: azhopConfig.deployer.vm_size
|
|
osdisksku: 'Standard_LRS'
|
|
image: 'ubuntu'
|
|
pip: enablePublicIP
|
|
sshPort: incomingSSHPort
|
|
asgs: union(
|
|
[ 'asg-ssh', 'asg-jumpbox', 'asg-deployer' ],
|
|
deployGrafana ? ['asg-telegraf'] : []
|
|
)
|
|
deploy_script: replace(replace(loadTextContent('install.sh'), '__INSERT_AZHOP_BRANCH__', branchName), '__SSH_PORT__', string(incomingSSHPort))
|
|
identity: {
|
|
keyvault: {
|
|
secret_permissions: [ 'All' ]
|
|
}
|
|
roles: [
|
|
'Contributor'
|
|
'UserAccessAdministrator'
|
|
]
|
|
}
|
|
}
|
|
} : {},
|
|
highAvailabilityForAD ? {
|
|
ad2: {
|
|
subnet: 'ad'
|
|
windows: true
|
|
ahub: contains(azhopConfig.ad, 'hybrid_benefit') ? azhopConfig.ad.hybrid_benefit : false
|
|
name: vmNamesMap.ad2
|
|
sku: azhopConfig.ad.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'win_base'
|
|
asgs: [ 'asg-ad', 'asg-rdp', 'asg-ad-client' ]
|
|
}
|
|
} : {} ,
|
|
deployJumpbox ? {
|
|
jumpbox: {
|
|
subnet: 'frontend'
|
|
name: vmNamesMap.jumpbox
|
|
sku: azhopConfig.jumpbox.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'linux_base'
|
|
pip: enablePublicIP
|
|
sshPort: incomingSSHPort
|
|
asgs: union(
|
|
[ 'asg-ssh', 'asg-jumpbox' ],
|
|
deployGrafana ? ['asg-telegraf'] : []
|
|
)
|
|
deploy_script: incomingSSHPort != 22 ? replace(loadTextContent('jumpbox.yml'), '__SSH_PORT__', string(incomingSSHPort)) : ''
|
|
}
|
|
} : {},
|
|
deployGrafana ? {
|
|
grafana: {
|
|
subnet: 'admin'
|
|
name: vmNamesMap.grafana
|
|
sku: azhopConfig.grafana.vm_size
|
|
osdisksku: 'StandardSSD_LRS'
|
|
image: 'linux_base'
|
|
asgs: union (
|
|
[ 'asg-ssh', 'asg-grafana', 'asg-telegraf', 'asg-nfs-client' ],
|
|
(userAuth == 'ad') ? ['asg-ad-client'] : []
|
|
)
|
|
}
|
|
} : {}
|
|
)
|
|
|
|
asg_names: union([ 'asg-ssh', 'asg-jumpbox', 'asg-sched', 'asg-cyclecloud', 'asg-cyclecloud-client', 'asg-nfs-client' ],
|
|
deployLustre ? [ 'asg-lustre-client' ] : [],
|
|
deployGrafana ? [ 'asg-grafana', 'asg-telegraf' ] : [],
|
|
(userAuth == 'ad') ? ['asg-rdp', 'asg-ad', 'asg-ad-client'] : [],
|
|
deployOnDemand ? ['asg-ondemand']: [],
|
|
createDatabase ? ['asg-mysql-client']: [],
|
|
deployDeployer || cycleCloudAsDeployer ? ['asg-deployer']: []
|
|
)
|
|
|
|
service_ports: {
|
|
All: ['0-65535']
|
|
Bastion: (incomingSSHPort == 22) ? ['22', '3389'] : ['22', string(incomingSSHPort), '3389']
|
|
Web: ['443', '80']
|
|
Ssh: ['22']
|
|
HubSsh: [string(incomingSSHPort)]
|
|
// DNS, Kerberos, RpcMapper, Ldap, Smb, KerberosPass, LdapSsl, LdapGc, LdapGcSsl, AD Web Services, RpcSam
|
|
DomainControlerTcp: ['53', '88', '135', '389', '445', '464', '636', '3268', '3269', '9389', '49152-65535']
|
|
// DNS, Kerberos, W32Time, NetBIOS, Ldap, KerberosPass, LdapSsl
|
|
DomainControlerUdp: ['53', '88', '123', '138', '389', '464', '636']
|
|
// Web, NoVNC, WebSockify
|
|
NoVnc: ['80', '443', '5900-5910', '61001-61010']
|
|
Dns: ['53']
|
|
Rdp: ['3389']
|
|
// Pbs: ['6200', '15001-15009', '17001', '32768-61000']
|
|
// Slurm: ['6817-6819']
|
|
Shed: (queue_manager == 'slurm') ? ['6817-6819', '59000-61000'] : ['6200', '15001-15009', '17001', '32768-61000']
|
|
Lustre: ['988', '1019-1023']
|
|
Nfs: ['111', '635', '2049', '4045', '4046']
|
|
SMB: ['445']
|
|
Telegraf: ['8086']
|
|
Grafana: ['3000']
|
|
// HTTPS, AMQP
|
|
CycleCloud: ['9443', '5672']
|
|
MySQL: ['3306', '33060']
|
|
WinRM: ['5985', '5986']
|
|
}
|
|
|
|
nsg_rules: {
|
|
default: {
|
|
//
|
|
// INBOUND RULES
|
|
//
|
|
|
|
// SSH internal rules
|
|
AllowSshFromJumpboxIn : ['320', 'Inbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-jumpbox', 'asg', 'asg-ssh']
|
|
AllowSshFromComputeIn : ['330', 'Inbound', 'Allow', 'Tcp', 'Ssh', 'subnet', 'compute', 'asg', 'asg-ssh']
|
|
AllowSshToComputeIn : ['360', 'Inbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-ssh', 'subnet', 'compute']
|
|
|
|
// All communications inside compute subnet
|
|
AllowAllComputeComputeIn : ['365', 'Inbound', 'Allow', 'Tcp', 'All', 'subnet', 'compute', 'subnet', 'compute']
|
|
|
|
// Scheduler
|
|
AllowSchedIn : ['369', 'Inbound', 'Allow', '*', 'Shed', 'asg', 'asg-sched', 'asg', 'asg-sched']
|
|
// AllowPbsClientIn : ['370', 'Inbound', 'Allow', '*', 'Pbs', 'asg', 'asg-pbs-client', 'asg', 'asg-pbs']
|
|
AllowSchedComputeIn : ['380', 'Inbound', 'Allow', '*', 'Shed', 'asg', 'asg-sched', 'subnet', 'compute']
|
|
// AllowComputePbsClientIn : ['390', 'Inbound', 'Allow', '*', 'Pbs', 'subnet', 'compute', 'asg', 'asg-pbs-client']
|
|
AllowComputeSchedIn : ['400', 'Inbound', 'Allow', '*', 'Shed', 'subnet', 'compute', 'asg', 'asg-sched']
|
|
// AllowComputeComputeSchedIn : ['401', 'Inbound', 'Allow', '*', 'Shed', 'subnet', 'compute', 'subnet', 'compute']
|
|
|
|
// CycleCloud
|
|
AllowCycleClientIn : ['450', 'Inbound', 'Allow', 'Tcp', 'CycleCloud', 'asg', 'asg-cyclecloud-client', 'asg', 'asg-cyclecloud']
|
|
AllowCycleClientComputeIn : ['460', 'Inbound', 'Allow', 'Tcp', 'CycleCloud', 'subnet', 'compute', 'asg', 'asg-cyclecloud']
|
|
AllowCycleServerIn : ['465', 'Inbound', 'Allow', 'Tcp', 'CycleCloud', 'asg', 'asg-cyclecloud', 'asg', 'asg-cyclecloud-client']
|
|
|
|
// Deny all remaining traffic
|
|
DenyVnetInbound : ['3100', 'Inbound', 'Deny', '*', 'All', 'tag', 'VirtualNetwork', 'tag', 'VirtualNetwork']
|
|
|
|
//
|
|
// Outbound
|
|
//
|
|
|
|
// CycleCloud
|
|
AllowCycleServerOut : ['300', 'Outbound', 'Allow', 'Tcp', 'CycleCloud', 'asg', 'asg-cyclecloud', 'asg', 'asg-cyclecloud-client']
|
|
AllowCycleClientOut : ['310', 'Outbound', 'Allow', 'Tcp', 'CycleCloud', 'asg', 'asg-cyclecloud-client', 'asg', 'asg-cyclecloud']
|
|
AllowComputeCycleClientIn : ['320', 'Outbound', 'Allow', 'Tcp', 'CycleCloud', 'subnet', 'compute', 'asg', 'asg-cyclecloud']
|
|
|
|
// Scheduler
|
|
AllowSchedOut : ['340', 'Outbound', 'Allow', '*', 'Shed', 'asg', 'asg-sched', 'asg', 'asg-sched']
|
|
// AllowPbsClientOut : ['350', 'Outbound', 'Allow', '*', 'Pbs', 'asg', 'asg-pbs-client', 'asg', 'asg-pbs']
|
|
AllowSchedComputeOut : ['360', 'Outbound', 'Allow', '*', 'Shed', 'asg', 'asg-sched', 'subnet', 'compute']
|
|
AllowComputeSchedOut : ['370', 'Outbound', 'Allow', '*', 'Shed', 'subnet', 'compute', 'asg', 'asg-sched']
|
|
//AllowComputePbsClientOut : ['380', 'Outbound', 'Allow', '*', 'Pbs', 'subnet', 'compute', 'asg', 'asg-pbs-client']
|
|
// AllowComputeComputeSchedOut : ['381', 'Outbound', 'Allow', '*', 'Shed', 'subnet', 'compute', 'subnet', 'compute']
|
|
|
|
// SSH internal rules
|
|
AllowSshFromJumpboxOut : ['490', 'Outbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-jumpbox', 'asg', 'asg-ssh']
|
|
AllowSshComputeOut : ['500', 'Outbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-ssh', 'subnet', 'compute']
|
|
AllowSshFromComputeOut : ['530', 'Outbound', 'Allow', 'Tcp', 'Ssh', 'subnet', 'compute', 'asg', 'asg-ssh']
|
|
|
|
// All communications inside compute subnet
|
|
AllowAllComputeComputeOut : ['540', 'Outbound', 'Allow', 'Tcp', 'All', 'subnet', 'compute', 'subnet', 'compute']
|
|
|
|
// Admin and Deployment
|
|
AllowDnsOut : ['590', 'Outbound', 'Allow', '*', 'Dns', 'tag', 'VirtualNetwork', 'tag', 'VirtualNetwork']
|
|
|
|
// Deny all remaining traffic and allow Internet access
|
|
AllowInternetOutBound : ['3000', 'Outbound', 'Allow', 'Tcp', 'All', 'tag', 'VirtualNetwork', 'tag', 'Internet']
|
|
DenyVnetOutbound : ['3100', 'Outbound', 'Deny', '*', 'All', 'tag', 'VirtualNetwork', 'tag', 'VirtualNetwork']
|
|
}
|
|
internet: {
|
|
AllowInternetSshIn : ['200', 'Inbound', 'Allow', 'Tcp', 'HubSsh', 'tag', 'Internet', 'asg', 'asg-jumpbox']
|
|
AllowInternetHttpIn : ['210', 'Inbound', 'Allow', 'Tcp', 'Web', 'tag', 'Internet', 'subnet', 'frontend']
|
|
}
|
|
hub: {
|
|
AllowHubSshIn : ['200', 'Inbound', 'Allow', 'Tcp', 'HubSsh', 'tag', 'VirtualNetwork', 'tag', 'VirtualNetwork']
|
|
AllowHubHttpIn : ['210', 'Inbound', 'Allow', 'Tcp', 'Web', 'tag', 'VirtualNetwork', 'tag', 'VirtualNetwork']
|
|
}
|
|
ad: {
|
|
// Inbound
|
|
// AD communication
|
|
AllowAdServerTcpIn : ['220', 'Inbound', 'Allow', 'Tcp', 'DomainControlerTcp', nsgTargetForDC.type, nsgTargetForDC.target, 'asg', 'asg-ad-client']
|
|
AllowAdServerUdpIn : ['230', 'Inbound', 'Allow', 'Udp', 'DomainControlerUdp', nsgTargetForDC.type, nsgTargetForDC.target, 'asg', 'asg-ad-client']
|
|
AllowAdClientTcpIn : ['240', 'Inbound', 'Allow', 'Tcp', 'DomainControlerTcp', 'asg', 'asg-ad-client', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdClientUdpIn : ['250', 'Inbound', 'Allow', 'Udp', 'DomainControlerUdp', 'asg', 'asg-ad-client', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdServerComputeTcpIn : ['260', 'Inbound', 'Allow', 'Tcp', 'DomainControlerTcp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'compute']
|
|
AllowAdServerComputeUdpIn : ['270', 'Inbound', 'Allow', 'Udp', 'DomainControlerUdp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'compute']
|
|
AllowAdClientComputeTcpIn : ['280', 'Inbound', 'Allow', 'Tcp', 'DomainControlerTcp', 'subnet', 'compute', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdClientComputeUdpIn : ['290', 'Inbound', 'Allow', 'Udp', 'DomainControlerUdp', 'subnet', 'compute', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowWinRMIn : ['520', 'Inbound', 'Allow', 'Tcp', 'WinRM', 'asg', 'asg-jumpbox', 'asg', 'asg-rdp']
|
|
AllowRdpIn : ['550', 'Inbound', 'Allow', 'Tcp', 'Rdp', 'asg', 'asg-jumpbox', 'asg', 'asg-rdp']
|
|
// Outbound
|
|
// AD communication
|
|
AllowAdClientTcpOut : ['200', 'Outbound', 'Allow', 'Tcp', 'DomainControlerTcp', 'asg', 'asg-ad-client', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdClientUdpOut : ['210', 'Outbound', 'Allow', 'Udp', 'DomainControlerUdp', 'asg', 'asg-ad-client', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdClientComputeTcpOut : ['220', 'Outbound', 'Allow', 'Tcp', 'DomainControlerTcp', 'subnet', 'compute', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdClientComputeUdpOut : ['230', 'Outbound', 'Allow', 'Udp', 'DomainControlerUdp', 'subnet', 'compute', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdServerTcpOut : ['240', 'Outbound', 'Allow', 'Tcp', 'DomainControlerTcp', nsgTargetForDC.type, nsgTargetForDC.target, 'asg', 'asg-ad-client']
|
|
AllowAdServerUdpOut : ['250', 'Outbound', 'Allow', 'Udp', 'DomainControlerUdp', nsgTargetForDC.type, nsgTargetForDC.target, 'asg', 'asg-ad-client']
|
|
AllowAdServerComputeTcpOut : ['260', 'Outbound', 'Allow', 'Tcp', 'DomainControlerTcp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'compute']
|
|
AllowAdServerComputeUdpOut : ['270', 'Outbound', 'Allow', 'Udp', 'DomainControlerUdp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'compute']
|
|
AllowRdpOut : ['570', 'Outbound', 'Allow', 'Tcp', 'Rdp', 'asg', 'asg-jumpbox', 'asg', 'asg-rdp']
|
|
AllowWinRMOut : ['580', 'Outbound', 'Allow', 'Tcp', 'WinRM', 'asg', 'asg-jumpbox', 'asg', 'asg-rdp']
|
|
}
|
|
ondemand: {
|
|
// Inbound
|
|
//AllowComputeSlurmIn : ['405', 'Inbound', 'Allow', '*', 'Slurmd', 'asg', 'asg-ondemand', 'subnet', 'compute']
|
|
AllowCycleWebIn : ['440', 'Inbound', 'Allow', 'Tcp', 'Web', 'asg', 'asg-ondemand', 'asg', 'asg-cyclecloud']
|
|
AllowComputeNoVncIn : ['470', 'Inbound', 'Allow', 'Tcp', 'NoVnc', 'subnet', 'compute', 'asg', 'asg-ondemand']
|
|
AllowNoVncComputeIn : ['480', 'Inbound', 'Allow', 'Tcp', 'NoVnc', 'asg', 'asg-ondemand', 'subnet', 'compute']
|
|
// Not sure if this is really needed. Why opening web port from deployer to ondemand ?
|
|
// AllowWebDeployerIn : ['595', 'Inbound', 'Allow', 'Tcp', 'Web', 'asg', 'asg-deployer', 'asg', 'asg-ondemand']
|
|
// Outbound
|
|
AllowCycleWebOut : ['330', 'Outbound', 'Allow', 'Tcp', 'Web', 'asg', 'asg-ondemand', 'asg', 'asg-cyclecloud']
|
|
//AllowSlurmComputeOut : ['385', 'Outbound', 'Allow', '*', 'Slurmd', 'asg', 'asg-ondemand', 'subnet', 'compute']
|
|
AllowComputeNoVncOut : ['550', 'Outbound', 'Allow', 'Tcp', 'NoVnc', 'subnet', 'compute', 'asg', 'asg-ondemand']
|
|
AllowNoVncComputeOut : ['560', 'Outbound', 'Allow', 'Tcp', 'NoVnc', 'asg', 'asg-ondemand', 'subnet', 'compute']
|
|
// AllowWebDeployerOut : ['595', 'Outbound', 'Allow', 'Tcp', 'Web', 'asg', 'asg-deployer', 'asg', 'asg-ondemand']
|
|
}
|
|
mysql: {
|
|
// Inbound
|
|
AllowMySQLIn : ['700', 'Inbound', 'Allow', 'Tcp', 'MySQL', 'asg', 'asg-mysql-client', 'subnet', 'database']
|
|
// Outbound
|
|
AllowMySQLOut : ['700', 'Outbound', 'Allow', 'Tcp', 'MySQL', 'asg', 'asg-mysql-client', 'subnet', 'database']
|
|
}
|
|
anf: {
|
|
// Inbound
|
|
AllowNfsIn : ['434', 'Inbound', 'Allow', '*', 'Nfs', 'asg', 'asg-nfs-client', 'subnet', 'netapp']
|
|
AllowNfsComputeIn : ['435', 'Inbound', 'Allow', '*', 'Nfs', 'subnet', 'compute', 'subnet', 'netapp']
|
|
// Outbound
|
|
AllowNfsOut : ['440', 'Outbound', 'Allow', '*', 'Nfs', 'asg', 'asg-nfs-client', 'subnet', 'netapp']
|
|
AllowNfsComputeOut : ['450', 'Outbound', 'Allow', '*', 'Nfs', 'subnet', 'compute', 'subnet', 'netapp']
|
|
}
|
|
ad_anf: {
|
|
// Inbound
|
|
AllowAdServerNetappTcpIn : ['300', 'Inbound', 'Allow', 'Tcp', 'DomainControlerTcp', 'subnet', 'netapp', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
AllowAdServerNetappUdpIn : ['310', 'Inbound', 'Allow', 'Udp', 'DomainControlerUdp', 'subnet', 'netapp', nsgTargetForDC.type, nsgTargetForDC.target]
|
|
// Outbound
|
|
AllowAdServerNetappTcpOut : ['280', 'Outbound', 'Allow', 'Tcp', 'DomainControlerTcp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'netapp']
|
|
AllowAdServerNetappUdpOut : ['290', 'Outbound', 'Allow', 'Udp', 'DomainControlerUdp', nsgTargetForDC.type, nsgTargetForDC.target, 'subnet', 'netapp']
|
|
}
|
|
lustre: {
|
|
// Inbound
|
|
AllowLustreClientIn : ['410', 'Inbound', 'Allow', 'Tcp', 'Lustre', 'asg', 'asg-lustre-client', 'subnet', 'lustre']
|
|
AllowLustreClientComputeIn : ['420', 'Inbound', 'Allow', 'Tcp', 'Lustre', 'subnet', 'compute', 'subnet', 'lustre']
|
|
AllowLustreSubnetAnyInbound : ['430', 'Inbound', 'Allow', '*', 'All', 'subnet', 'lustre', 'subnet', 'lustre']
|
|
// Outbound
|
|
AllowAzureCloudServiceAccess: ['400', 'Outbound', 'Allow', '*', 'All', 'tag', 'VirtualNetwork', 'tag', 'AzureCloud']
|
|
AllowLustreClientOut : ['410', 'Outbound', 'Allow', 'Tcp', 'Lustre', 'asg', 'asg-lustre-client', 'subnet', 'lustre']
|
|
AllowLustreClientComputeOut : ['420', 'Outbound', 'Allow', 'Tcp', 'Lustre', 'subnet', 'compute', 'subnet', 'lustre']
|
|
AllowLustreSubnetAnyOutbound: ['430', 'Outbound', 'Allow', '*', 'All', 'subnet', 'lustre', 'subnet', 'lustre']
|
|
}
|
|
bastion: {
|
|
AllowBastionIn : ['530', 'Inbound' , 'Allow', 'Tcp', 'Bastion', 'subnet', 'bastion', 'tag', 'VirtualNetwork']
|
|
AllowBastionOut : ['531', 'Outbound', 'Allow', 'Tcp', 'Bastion', 'subnet', 'bastion', 'tag', 'VirtualNetwork']
|
|
}
|
|
gateway: {
|
|
AllowInternalWebUsersIn : ['540', 'Inbound', 'Allow', 'Tcp', 'Web', 'subnet', 'gateway', 'asg', 'asg-ondemand']
|
|
}
|
|
grafana: {
|
|
// Telegraf / Grafana
|
|
// Inbound
|
|
AllowTelegrafIn : ['490', 'Inbound', 'Allow', 'Tcp', 'Telegraf', 'asg', 'asg-telegraf', 'asg', 'asg-grafana']
|
|
AllowComputeTelegrafIn : ['500', 'Inbound', 'Allow', 'Tcp', 'Telegraf', 'subnet', 'compute', 'asg', 'asg-grafana']
|
|
AllowGrafanaIn : ['510', 'Inbound', 'Allow', 'Tcp', 'Grafana', 'asg', 'asg-ondemand', 'asg', 'asg-grafana']
|
|
// Outbound
|
|
AllowTelegrafOut : ['460', 'Outbound', 'Allow', 'Tcp', 'Telegraf', 'asg', 'asg-telegraf', 'asg', 'asg-grafana']
|
|
AllowComputeTelegrafOut : ['470', 'Outbound', 'Allow', 'Tcp', 'Telegraf', 'subnet', 'compute', 'asg', 'asg-grafana']
|
|
AllowGrafanaOut : ['480', 'Outbound', 'Allow', 'Tcp', 'Grafana', 'asg', 'asg-ondemand', 'asg', 'asg-grafana']
|
|
}
|
|
deployer: {
|
|
// Inbound
|
|
AllowSshFromDeployerIn : ['340', 'Inbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-deployer', 'asg', 'asg-ssh']
|
|
AllowDeployerToPackerSshIn : ['350', 'Inbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-deployer', 'subnet', 'admin']
|
|
// Outbound
|
|
AllowSshDeployerOut : ['510', 'Outbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-deployer', 'asg', 'asg-ssh']
|
|
AllowSshDeployerPackerOut : ['520', 'Outbound', 'Allow', 'Tcp', 'Ssh', 'asg', 'asg-deployer', 'subnet', 'admin']
|
|
}
|
|
}
|
|
}
|
|
|
|
var vmItems = items(config.vms)
|
|
|
|
module azhopSecrets './secrets.bicep' = if (autogenerateSecrets) {
|
|
name: 'azhopSecrets'
|
|
params: {
|
|
location: location
|
|
kvName: autogenerateSecrets ? azhopKeyvault.outputs.keyvaultName : 'foo' // trick to avoid unreferenced resource for azhopKeyvaultSecrets
|
|
adminUser: config.admin_user
|
|
dbAdminUser: config.slurm.admin_user
|
|
identityId: autogenerateSecrets ? identity.id : '' // trick to avoid unreferenced resource for identity
|
|
}
|
|
}
|
|
|
|
resource kv 'Microsoft.KeyVault/vaults@2022-11-01' existing = if (autogenerateSecrets) {
|
|
name: azhopKeyvault.outputs.keyvaultName
|
|
}
|
|
|
|
module natgateway './natgateway.bicep' = if (config.nat_gateway.create) {
|
|
name: 'natgateway'
|
|
params: {
|
|
location: location
|
|
name: config.nat_gateway.name
|
|
}
|
|
}
|
|
|
|
var natGatewayId = config.nat_gateway.create ? natgateway.outputs.NATGatewayId : ''
|
|
var nsgRules = items(union(
|
|
config.nsg_rules.default,
|
|
(userAuth == 'ad') ? config.nsg_rules.ad : {},
|
|
(userAuth == 'ad') && config.anf.create ? config.nsg_rules.ad_anf : {},
|
|
config.public_ip ? config.nsg_rules.internet : config.nsg_rules.hub,
|
|
config.deploy_bastion ? config.nsg_rules.bastion : {},
|
|
config.deploy_gateway ? config.nsg_rules.gateway : {},
|
|
config.anf.create ? config.nsg_rules.anf : {},
|
|
config.deploy_lustre ? config.nsg_rules.lustre : {},
|
|
config.deploy_grafana ? config.nsg_rules.grafana : {},
|
|
config.deploy_ondemand ? config.nsg_rules.ondemand: {},
|
|
createDatabase ? config.nsg_rules.mysql: {},
|
|
deployDeployer ? config.nsg_rules.deployer: {}
|
|
))
|
|
|
|
module azhopNetwork './network.bicep' = {
|
|
name: 'azhopNetwork'
|
|
params: {
|
|
location: location
|
|
vnet: config.vnet
|
|
asgNames: config.asg_names
|
|
servicePorts: config.service_ports
|
|
nsgRules: nsgRules
|
|
peerings: config.vnet.peerings
|
|
natGatewayId: natGatewayId
|
|
}
|
|
}
|
|
|
|
output vnetId string = azhopNetwork.outputs.vnetId
|
|
|
|
var subnetIds = azhopNetwork.outputs.subnetIds
|
|
var asgNameToIdLookup = reduce(azhopNetwork.outputs.asgIds, {}, (cur, next) => union(cur, next))
|
|
|
|
|
|
module azhopBastion './bastion.bicep' = if (config.deploy_bastion) {
|
|
name: 'azhopBastion'
|
|
params: {
|
|
location: location
|
|
subnetId: subnetIds.bastion
|
|
}
|
|
}
|
|
|
|
module azhopVm './vm.bicep' = [ for vm in vmItems: {
|
|
name: 'azhopVm${vm.key}'
|
|
params: {
|
|
location: location
|
|
name: contains(vm.value, 'name') ? vm.value.name : vm.key
|
|
vm: vm.value
|
|
image: config.images[vm.value.image]
|
|
subnetId: subnetIds[vm.value.subnet]
|
|
adminUser: config.admin_user
|
|
adminPassword: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.adminPassword) : adminPassword
|
|
adminSshPublicKey: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.adminSshPublicKey) : adminSshPublicKey
|
|
asgIds: asgNameToIdLookup
|
|
}
|
|
}]
|
|
|
|
// Assign roles to VMs for which roles have been specified
|
|
module azhopRoleAssignements './roleAssignments.bicep' = [ for vm in vmItems: if (contains(vm.value, 'identity') && contains(vm.value.identity, 'roles')) {
|
|
name: 'azhopRoleFor${vm.key}'
|
|
params: {
|
|
name: vm.key
|
|
roles: vm.value.identity.roles
|
|
principalId: azhopVm[indexOf(map(vmItems, item => item.key), vm.key)].outputs.principalId
|
|
}
|
|
}]
|
|
|
|
resource identity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (autogenerateSecrets) {
|
|
name: 'deployScriptIdentity'
|
|
location: location
|
|
}
|
|
|
|
resource computemi 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if(createComputeMI) {
|
|
name: computeMIname
|
|
location: location
|
|
}
|
|
|
|
module kvAccessPoliciesSecrets './kv_access_policies.bicep' = if (autogenerateSecrets) {
|
|
name: 'kvAccessPoliciesSecrets'
|
|
params: {
|
|
vaultName: autogenerateSecrets ? azhopKeyvault.outputs.keyvaultName : 'foo' // trick to avoid unreferenced resource for azhopKeyvaultSecrets
|
|
secret_permissions: ['Set']
|
|
principalId: autogenerateSecrets ? identity.properties.principalId : ''
|
|
}
|
|
}
|
|
|
|
module azhopKeyvault './keyvault.bicep' = {
|
|
name: 'azhopKeyvault'
|
|
params: {
|
|
location: location
|
|
kvName: config.key_vault_name
|
|
subnetId: subnetIds.admin
|
|
keyvaultReaderOids: config.keyvault_readers
|
|
lockDownNetwork: config.lock_down_network.enforce
|
|
allowableIps: config.lock_down_network.grant_access_from
|
|
keyvaultOwnerId: loggedUserObjectId
|
|
}
|
|
}
|
|
|
|
module kvAccessPolicies './kv_access_policies.bicep' = [ for vm in vmItems: if (contains(vm.value, 'identity') && contains(vm.value.identity, 'keyvault')) {
|
|
name: 'kvAccessPolicies${vm.key}'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
secret_permissions: contains(vm.value.identity.keyvault, 'secret_permissions') ? vm.value.identity.keyvault.secret_permissions : []
|
|
principalId: azhopVm[indexOf(map(vmItems, item => item.key), vm.key)].outputs.principalId
|
|
}
|
|
}]
|
|
|
|
|
|
module kvSecretAdminPassword './kv_secrets.bicep' = if (!autogenerateSecrets) {
|
|
name: 'kvSecrets-admin-password'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.admin_user}-password'
|
|
value: adminPassword
|
|
}
|
|
}
|
|
|
|
module kvSecretAdminPubKey './kv_secrets.bicep' = if (!autogenerateSecrets) {
|
|
name: 'kvSecrets-admin-pubkey'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.admin_user}-pubkey'
|
|
value: adminSshPublicKey
|
|
}
|
|
}
|
|
|
|
module kvSecretAdminPrivKey './kv_secrets.bicep' = if (!autogenerateSecrets) {
|
|
name: 'kvSecrets-admin-privkey'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.admin_user}-privkey'
|
|
value: adminSshPrivateKey
|
|
}
|
|
}
|
|
|
|
module kvSecretDBPassword './kv_secrets.bicep' = if (!autogenerateSecrets && createDatabase) {
|
|
name: 'kvSecrets-db-password'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.slurm.admin_user}-password'
|
|
value: databaseAdminPassword
|
|
}
|
|
}
|
|
|
|
// Domain join password when deploying AD will be stored in the keyvault
|
|
module kvSecretDomainJoin './kv_secrets.bicep' = if (createAD) {
|
|
name: 'kvSecrets-domain-join'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.domain.domain_join_user.username}-password'
|
|
value: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.adminPassword) : adminPassword
|
|
}
|
|
}
|
|
|
|
// Domain join password when using an existing AD will be retrieved from the keyvault specified in config and stored in our KV
|
|
resource domainJoinUserKV 'Microsoft.KeyVault/vaults@2021-10-01' existing = if (useExistingAD) {
|
|
name: '${config.domain.domain_join_user.password_key_vault_name}'
|
|
scope: resourceGroup(config.domain.domain_join_user.password_key_vault_resource_group_name)
|
|
}
|
|
module kvSecretExistingDomainJoin './kv_secrets.bicep' = if (useExistingAD) {
|
|
name: 'kvSecrets-existing-domain-join'
|
|
params: {
|
|
vaultName: azhopKeyvault.outputs.keyvaultName
|
|
name: '${config.domain.domain_join_user.username}-password'
|
|
value: domainJoinUserKV.getSecret(config.domain.domain_join_user.password_key_vault_secret_name)
|
|
}
|
|
}
|
|
|
|
module azhopStorage './storage.bicep' = {
|
|
name: 'azhopStorage'
|
|
params:{
|
|
location: location
|
|
saName: config.storage_account_name
|
|
lockDownNetwork: config.lock_down_network.enforce
|
|
allowableIps: config.lock_down_network.grant_access_from
|
|
subnetIds: [ subnetIds.admin, subnetIds.compute ]
|
|
}
|
|
}
|
|
|
|
module azhopSig './sig.bicep' = if (config.deploy_sig) {
|
|
name: 'azhopSig'
|
|
params: {
|
|
location: location
|
|
resourcePostfix: resourcePostfix
|
|
}
|
|
}
|
|
|
|
// module azhopMariaDB './mariadb.bicep' = if (createDatabase) {
|
|
// name: 'azhopMariaDB'
|
|
// params: {
|
|
// location: location
|
|
// mariaDbName: config.mariadb_name
|
|
// adminUser: config.slurm.admin_user
|
|
// adminPassword: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.databaseAdminPassword) : databaseAdminPassword
|
|
// adminSubnetId: subnetIds.admin
|
|
// vnetId: azhopNetwork.outputs.vnetId
|
|
// sslEnforcement: true
|
|
// }
|
|
// }
|
|
|
|
module mySQL './mysql.bicep' = if (createDatabase) {
|
|
name: 'mySQLDB'
|
|
params: {
|
|
location: location
|
|
Name: config.db_name
|
|
adminUser: config.slurm.admin_user
|
|
adminPassword: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.databaseAdminPassword) : databaseAdminPassword
|
|
subnetId: subnetIds.database
|
|
}
|
|
}
|
|
|
|
module azhopTelemetry './telemetry.bicep' = {
|
|
name: 'azhopTelemetry'
|
|
}
|
|
|
|
module azhopVpnGateway './vpngateway.bicep' = if (config.deploy_gateway) {
|
|
name: 'azhopVpnGateway'
|
|
params: {
|
|
location: location
|
|
subnetId: subnetIds.gateway
|
|
}
|
|
}
|
|
|
|
module azhopAmlfs './amlfs.bicep' = if (deployLustre) {
|
|
name: 'azhopAmlfs'
|
|
params: {
|
|
location: location
|
|
name: 'amlfs${resourcePostfix}'
|
|
subnetId: subnetIds.lustre
|
|
sku: config.lustre.sku
|
|
capacity: config.lustre.capacity
|
|
}
|
|
}
|
|
|
|
module azhopAnf './anf.bicep' = if (config.anf.create) {
|
|
name: 'azhopAnf'
|
|
params: {
|
|
location: location
|
|
resourcePostfix: resourcePostfix
|
|
dualProtocol: config.anf.dual_protocol
|
|
subnetId: subnetIds.netapp
|
|
adUser: config.admin_user
|
|
adPassword: autogenerateSecrets ? kv.getSecret(azhopSecrets.outputs.secrets.adminPassword) : adminPassword
|
|
adDns: adIp
|
|
serviceLevel: config.anf.service_level
|
|
sizeGB: config.anf.size_gb
|
|
}
|
|
}
|
|
|
|
module azhopNfsFiles './nfsfiles.bicep' = if (config.azurefiles.create ) {
|
|
name: 'azhopNfsFiles'
|
|
params: {
|
|
location: location
|
|
resourcePostfix: resourcePostfix
|
|
allowedSubnetIds: [ subnetIds.admin, subnetIds.compute, subnetIds.frontend ]
|
|
sizeGB: config.azurefiles.size_gb
|
|
}
|
|
}
|
|
|
|
module azhopPrivateZone './privatezone.bicep' = if (createAD || useExistingAD || config.private_dns.create) {
|
|
name: 'azhopPrivateZone'
|
|
params: {
|
|
privateDnsZoneName: config.domain.name
|
|
vnetId: azhopNetwork.outputs.vnetId
|
|
registrationEnabled: config.private_dns.registration_enabled
|
|
}
|
|
}
|
|
|
|
// list of DC VMs. The first one will be considered the default PDC (for DNS registration)
|
|
// Trick to get the index of the DC VM in the vmItems array, to workaround a bug in bicep 0.14.85 as it throws an error when using indexOf(map(vmItems, item => item.key), 'ad2')
|
|
var adIndex = createAD ? indexOf(map(vmItems, item => item.key), 'ad') : 0
|
|
var adIp = createAD ? azhopVm[adIndex].outputs.privateIp : ''
|
|
var ad2Index = createAD && highAvailabilityForAD ? indexOf(map(vmItems, item => item.key), 'ad2') : 0
|
|
var ad2Ip = createAD ? azhopVm[ad2Index].outputs.privateIp : ''
|
|
var domain_controller_ip_addresses = useExistingAD && contains(azhopConfig, 'domain') && contains(azhopConfig.domain, 'existing_dc_details') ? azhopConfig.domain.existing_dc_details.domain_controller_ip_addresses : []
|
|
var dcIps = createAD ? (! highAvailabilityForAD ? [adIp] : [adIp, ad2Ip]) : domain_controller_ip_addresses
|
|
module azhopADRecords './privatezone_records.bicep' = if (createAD || useExistingAD) {
|
|
name: 'azhopADRecords'
|
|
params: {
|
|
privateDnsZoneName: config.domain.name
|
|
adVmNames: config.domain.domain_controlers
|
|
adVmIps: dcIps
|
|
}
|
|
}
|
|
|
|
output ccportalPrincipalId string = azhopVm[indexOf(map(vmItems, item => item.key), 'ccportal')].outputs.principalId
|
|
|
|
output keyvaultName string = azhopKeyvault.outputs.keyvaultName
|
|
|
|
// Our input file is also the deployment output
|
|
output azhopConfig object = azhopConfig
|
|
|
|
var envNameToCloudMap = {
|
|
AzureCloud: 'AZUREPUBLICCLOUD'
|
|
AzureUSGovernment: 'AZUREUSGOVERNMENT'
|
|
AzureGermanCloud: 'AZUREGERMANCLOUD'
|
|
AzureChinaCloud: 'AZURECHINACLOUD'
|
|
}
|
|
|
|
var kvSuffix = environment().suffixes.keyvaultDns
|
|
|
|
output azhopGlobalConfig object = union(
|
|
{
|
|
global_cc_storage : config.storage_account_name
|
|
compute_subnetid : '${azhopResourceGroupName}/${config.vnet.name}/${config.vnet.subnets.compute.name}'
|
|
global_config_file : '/az-hop/config.yml'
|
|
ad_join_user : config.domain.domain_join_user.username
|
|
domain_name : config.domain.name
|
|
ldap_server : '${config.domain.ldap_server}.${config.domain.name}'
|
|
homedir_mountpoint : config.homedir_mountpoint
|
|
ondemand_fqdn : deployOnDemand ? (config.public_ip ? azhopVm[indexOf(map(vmItems, item => item.key), 'ondemand')].outputs.fqdn : azhopVm[indexOf(map(vmItems, item => item.key), 'ondemand')].outputs.privateIp) : ''
|
|
ansible_ssh_private_key_file : '${config.admin_user}_id_rsa'
|
|
subscription_id : subscription().subscriptionId
|
|
tenant_id : subscription().tenantId
|
|
key_vault : config.key_vault_name
|
|
sig_name : (config.deploy_sig) ? 'azhop_${resourcePostfix}' : ''
|
|
lustre_hsm_storage_account : config.storage_account_name
|
|
lustre_hsm_storage_container : 'lustre'
|
|
database_fqdn : createDatabase ? mySQL.outputs.fqdn : ''
|
|
database_user : config.slurm.admin_user
|
|
azure_environment : envNameToCloudMap[environment().name]
|
|
key_vault_suffix : substring(kvSuffix, 1, length(kvSuffix) - 1) // vault.azure.net - remove leading dot from env
|
|
blob_storage_suffix : 'blob.${environment().suffixes.storage}' // blob.core.windows.net
|
|
jumpbox_ssh_port : incomingSSHPort
|
|
},
|
|
createComputeMI ? {
|
|
compute_mi_id : resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', computemi.name)
|
|
}: {},
|
|
!empty(existingComputeMIrg) ? {
|
|
compute_mi_id : resourceId(existingComputeMIrg,'Microsoft.ManagedIdentity/userAssignedIdentities', computeMIname)
|
|
}: {},
|
|
config.homedir_type == 'anf' ? {
|
|
anf_home_ip : azhopAnf.outputs.nfs_home_ip
|
|
anf_home_path : azhopAnf.outputs.nfs_home_path
|
|
anf_home_opts : azhopAnf.outputs.nfs_home_opts
|
|
} : {},
|
|
config.homedir_type == 'azurefiles' ? {
|
|
anf_home_ip : azhopNfsFiles.outputs.nfs_home_ip
|
|
anf_home_path : azhopNfsFiles.outputs.nfs_home_path
|
|
anf_home_opts : azhopNfsFiles.outputs.nfs_home_opts
|
|
} : {},
|
|
config.homedir_type == 'existing' ? {
|
|
anf_home_ip : azhopConfig.mounts.home.server
|
|
anf_home_path : azhopConfig.mounts.home.export
|
|
anf_home_opts : azhopConfig.mounts.home.options
|
|
} : {},
|
|
deployLustre ? {
|
|
lustre_mgs : azhopAmlfs.outputs.lustre_mgs
|
|
} : {}
|
|
)
|
|
|
|
var sshTunelIp = deployJumpbox ? ( config.public_ip ? azhopVm[indexOf(map(vmItems, item => item.key), 'jumpbox')].outputs.publicIp : azhopVm[indexOf(map(vmItems, item => item.key), 'jumpbox')].outputs.privateIp ) : ''
|
|
|
|
output azhopInventory object = {
|
|
all: {
|
|
hosts: union (
|
|
{
|
|
localhost: {
|
|
psrp_ssh_proxy: sshTunelIp
|
|
}
|
|
scheduler: {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'scheduler')].outputs.privateIp
|
|
}
|
|
ccportal: {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'ccportal')].outputs.privateIp
|
|
}
|
|
},
|
|
config.deploy_ondemand ? {
|
|
ondemand: {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'ondemand')].outputs.privateIp
|
|
}
|
|
} : {},
|
|
indexOf(map(vmItems, item => item.key), 'ad') >= 0 ? {
|
|
ad: {
|
|
ansible_host: adIp
|
|
ansible_connection: 'psrp'
|
|
ansible_psrp_protocol: 'http'
|
|
ansible_user: config.admin_user
|
|
ansible_password: '__ADMIN_PASSWORD__'
|
|
psrp_ssh_proxy: sshTunelIp
|
|
ansible_psrp_proxy: deployJumpbox ? 'socks5h://localhost:5985' : ''
|
|
}
|
|
} : {} ,
|
|
indexOf(map(vmItems, item => item.key), 'ad2') >= 0 ? {
|
|
ad2: {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'ad2')].outputs.privateIp
|
|
ansible_connection: 'psrp'
|
|
ansible_psrp_protocol: 'http'
|
|
ansible_user: config.admin_user
|
|
ansible_password: '__ADMIN_PASSWORD__'
|
|
psrp_ssh_proxy: sshTunelIp
|
|
ansible_psrp_proxy: deployJumpbox ? 'socks5h://localhost:5985' : ''
|
|
}
|
|
} : {} ,
|
|
deployJumpbox ? {
|
|
jumpbox : {
|
|
ansible_host: sshTunelIp
|
|
ansible_ssh_port: incomingSSHPort
|
|
ansible_ssh_common_args: ''
|
|
}
|
|
} : {},
|
|
deployDeployer ? {
|
|
deployer : {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'deployer')].outputs.privateIp
|
|
ansible_ssh_port: incomingSSHPort
|
|
ansible_ssh_common_args: ''
|
|
}
|
|
} : {},
|
|
config.deploy_grafana ? {
|
|
grafana: {
|
|
ansible_host: azhopVm[indexOf(map(vmItems, item => item.key), 'grafana')].outputs.privateIp
|
|
}
|
|
} : {}
|
|
)
|
|
vars: {
|
|
ansible_ssh_user: config.admin_user
|
|
ansible_ssh_common_args: deployJumpbox ? '-o ProxyCommand="ssh -i ${config.admin_user}_id_rsa -p ${incomingSSHPort} -W %h:%p ${config.admin_user}@${sshTunelIp}"' : ''
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
output azhopPackerOptions object = (config.deploy_sig) ? {
|
|
var_subscription_id: subscription().subscriptionId
|
|
var_resource_group: azhopResourceGroupName
|
|
var_location: location
|
|
var_sig_name: 'azhop_${resourcePostfix}'
|
|
var_private_virtual_network_with_public_ip: 'false'
|
|
var_virtual_network_name: config.vnet.name
|
|
var_virtual_network_subnet_name: config.vnet.subnets.compute.name
|
|
var_virtual_network_resource_group_name: azhopResourceGroupName
|
|
var_ssh_bastion_host: sshTunelIp
|
|
var_ssh_bastion_port: incomingSSHPort
|
|
var_ssh_bastion_username: config.admin_user
|
|
var_ssh_bastion_private_key_file: '../${config.admin_user}_id_rsa'
|
|
var_queue_manager: config.queue_manager
|
|
} : {}
|
|
|
|
var azhopConnectScript = format('''
|
|
#!/bin/bash
|
|
|
|
exec ssh -i {0}_id_rsa "$@"
|
|
|
|
''', config.admin_user)
|
|
|
|
var azhopSSHConnectScript = format('''
|
|
#!/bin/bash
|
|
case $1 in
|
|
cyclecloud)
|
|
echo go create tunnel to cyclecloud at https://localhost:9443/cyclecloud
|
|
ssh -i {0}_id_rsa -fN -L 9443:ccportal:9443 -p {1} {0}@{2}
|
|
;;
|
|
ad)
|
|
echo go create tunnel to ad with rdp to localhost:3390
|
|
ssh -i {0}_id_rsa -fN -L 3390:ad:3389 -p {1} {0}@{2}
|
|
;;
|
|
deployer|jumpbox)
|
|
ssh -i {0}_id_rsa -p {1} {0}@{2}
|
|
;;
|
|
*)
|
|
exec ssh -i {0}_id_rsa -o ProxyCommand="ssh -i {0}_id_rsa -p {1} -W %h:%p {0}@{2}" -o "User={0}" "$@"
|
|
;;
|
|
esac
|
|
''', config.admin_user, incomingSSHPort, sshTunelIp)
|
|
|
|
output azhopConnectScript string = deployDeployer ? azhopConnectScript : azhopSSHConnectScript
|
|
|
|
|
|
output azhopGetSecretScript string = format('''
|
|
#!/bin/bash
|
|
|
|
user=$1
|
|
# Because secret names are restricted to '^[0-9a-zA-Z-]+$' we need to remove all other characters
|
|
secret_name=$(echo $user-password | tr -dc 'a-zA-Z0-9-')
|
|
|
|
az keyvault secret show --vault-name {0} -n $secret_name --query "value" -o tsv
|
|
|
|
''', config.key_vault_name)
|