* add wrapper bicep file

* update pipeline

* fix nsg typo

* update dependency

* update dependency

* default kube version

* update resourcegroup name

* update pipeline

* update kv script

* moving hardcoded values into parameter file

* remove certificates from param file

* adding parameter file into pipeline

* remove default values

---------

Co-authored-by: Ayobami Ayodeji <ayobaami@hotmail.com>
Co-authored-by: Rainer Halanek <rahalan@microsoft.com>
Co-authored-by: Rainer Halanek <61878316+rahalan@users.noreply.github.com>
This commit is contained in:
Julian Peißker 2023-04-18 09:04:31 +02:00 коммит произвёл GitHub
Родитель 9d9497cae6
Коммит d4b2b49b57
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 407 добавлений и 65 удалений

70
.github/workflows/IaC-bicep-AKS.yml поставляемый
Просмотреть файл

@ -116,82 +116,34 @@ jobs:
tenant-id: ${{ secrets.AZURE_TENANT_ID }} tenant-id: ${{ secrets.AZURE_TENANT_ID }}
subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
- name: "Deploy Hub" - name: "Deploy AKS Landingzone"
id: hub id: akslz
uses: azure/arm-deploy@v1 uses: azure/arm-deploy@v1
with: with:
subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} subscriptionId: ${{ secrets.SUBSCRIPTION_ID }}
region: ${{ github.event.inputs.REGION }} region: ${{ github.event.inputs.REGION }}
scope: subscription scope: subscription
template: ./IaC/bicep/rg-hub/hub-default.bicep template: ./IaC/bicep/main.bicep
parameters: ./IaC/bicep/rg-hub/hub-default.parameters.json resourceGroupName=rg-enterprise-networking-hubs-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} parameters: ./IaC/bicep/main.parameters.json clusterAdminAadGroupObjectId=${{ github.event.inputs.clusterAdminAadGroupObjectId }} a0008NamespaceReaderAadGroupObjectId=${{ github.event.inputs.a0008NamespaceReaderAadGroupObjectId }} appGatewayListenerCertificate=${{ env.APP_GATEWAY_LISTENER_CERTIFICATE }} aksIngressControllerCertificate=${{ env.AKS_INGRESS_CONTROLLER_CERTIFICATE }}
failOnStdErr: false failOnStdErr: false
deploymentName: carml-hub-${{ github.event.inputs.REGION }} deploymentName: carml-aks-landingzone-${{ github.event.inputs.REGION }}
- name: "Deploy Spoke"
id: spoke
uses: azure/arm-deploy@v1
with:
subscriptionId: ${{ secrets.SUBSCRIPTION_ID }}
region: ${{ github.event.inputs.REGION }}
scope: subscription
template: ./IaC/bicep/rg-spoke/spoke.bicep
parameters: ./IaC/bicep/rg-spoke/spoke.parameters.json hubVnetResourceId=${{ steps.hub.outputs.hubVnetId }} hubLaWorkspaceResourceId=${{ steps.hub.outputs.hubLaWorkspaceResourceId }} hubFwResourceId=${{ steps.hub.outputs.hubFwResourceId }} resourceGroupName=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }}
failOnStdErr: false
deploymentName: carml-spoke-${{ github.event.inputs.REGION }}
- name: "Deploy Registry"
id: registry
uses: azure/arm-deploy@v1
with:
subscriptionId: ${{ secrets.SUBSCRIPTION_ID }}
region: ${{ github.event.inputs.REGION }}
scope: subscription
template: ./IaC/bicep/rg-spoke/acr.bicep
parameters: ./IaC/bicep/rg-spoke/acr.parameters.json targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }}
failOnStdErr: false
deploymentName: carml-registry-${{ github.event.inputs.REGION }}
# Import core images hosted in public container registries to be used during bootstrapping # Import core images hosted in public container registries to be used during bootstrapping
- name: "Import Images into ACR for flux" - name: "Import Images into ACR for flux"
id: image_import id: image_import
run: | run: |
az acr import --source docker.io/weaveworks/kured:1.10.1 -n ${{ steps.registry.outputs.containerRegistryName }} --force az acr import --source docker.io/weaveworks/kured:1.10.1 -n ${{ steps.akslz.outputs.containerRegistryName }} --force
az acr import --source docker.io/library/traefik:v2.8.1 -n ${{ steps.registry.outputs.containerRegistryName }} --force az acr import --source docker.io/library/traefik:v2.8.1 -n ${{ steps.akslz.outputs.containerRegistryName }} --force
- name: "Deploy Cluster prerequisites"
id: cluster_prereq
uses: azure/arm-deploy@v1
with:
subscriptionId: ${{ secrets.SUBSCRIPTION_ID }}
region: ${{ github.event.inputs.REGION }}
scope: subscription
template: ./IaC/bicep/rg-spoke/clusterprereq.bicep
parameters: ./IaC/bicep/rg-spoke/clusterprereq.parameters.json appGatewayListenerCertificate=${{ env.APP_GATEWAY_LISTENER_CERTIFICATE }} aksIngressControllerCertificate=${{ env.AKS_INGRESS_CONTROLLER_CERTIFICATE }} targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} vNetResourceGroup=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }}
failOnStdErr: false
deploymentName: carml-cluster-${{ github.event.inputs.REGION }}
- name: "Deploy Cluster"
id: cluster
uses: azure/arm-deploy@v1
with:
subscriptionId: ${{ secrets.SUBSCRIPTION_ID }}
region: ${{ github.event.inputs.REGION }}
scope: subscription
template: ./IaC/bicep/rg-spoke/cluster.bicep
parameters: ./IaC/bicep/rg-spoke/cluster.parameters.json targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} clusterAdminAadGroupObjectId=${{ github.event.inputs.clusterAdminAadGroupObjectId }} a0008NamespaceReaderAadGroupObjectId=${{ github.event.inputs.a0008NamespaceReaderAadGroupObjectId }} vNetResourceGroup=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }} gitOpsBootstrappingRepoBranch=${{ github.ref_name }}
failOnStdErr: false
deploymentName: carml-cluster-${{ github.event.inputs.REGION }}
# Temporary workaround until we figure out why steps.cluster.outputs.keyVaultName is blank # Temporary workaround until we figure out why steps.cluster.outputs.keyVaultName is blank
- name: "Get KeyVault Name" - name: "Get KeyVault Name"
id: akv_name id: akv_name
run: | run: |
export AKV_NAME=$(az deployment group list -g rg-BU0001A0008-${{ github.event.inputs.REGION }} -o table| grep 'kv-aks'|awk '{print $1}') export AKV_NAME=$(az keyvault list -g rg-bu0001a0008 -o table | grep "kv-aks" | awk '{print $2}')
echo "AKV_NAME=${AKV_NAME}" >> $GITHUB_ENV echo "AKV_NAME=${AKV_NAME}" >> $GITHUB_ENV
echo "AKV Name from bicep output is ${{ steps.cluster.outputs.keyVaultName }}" echo "AKV Name from bicep output is ${{ steps.akslz.outputs.keyVaultName }}"
echo "aksIngressControllerPodManagedIdentityResourceId from bicep output is ${{ steps.cluster.outputs.aksIngressControllerPodManagedIdentityResourceId }}" echo "aksIngressControllerPodManagedIdentityResourceId from bicep output is ${{ steps.akslz.outputs.aksIngressControllerPodManagedIdentityResourceId }}"
echo "To prove that this should work: hubVnetId is ${{ steps.hub.outputs.hubVnetId }}" echo "To prove that this should work: hubVnetId is ${{ steps.akslz.outputs.hubVnetId }}"
# Re-authentication is required for the commands that follow # Re-authentication is required for the commands that follow
- name: Azure Login - name: Azure Login

163
IaC/bicep/main.bicep Normal file
Просмотреть файл

@ -0,0 +1,163 @@
targetScope = 'subscription'
@description('Name of the hub resource group')
param hubResourceGroupName string
@description('Name of the spoke resource group')
param spokeResourceGroupName string
@description('Name of the AKS resource group')
param aksResourceGroupName string
@description('AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.')
param location string
@description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.')
param geoRedundancyLocation string
@description('Subnet address prefixes for all AKS clusters nodepools in all attached spokes to allow necessary outbound traffic through the firewall.')
@minLength(1)
param subnetIpAddressSpace array
@description('Optional. Array of Security Rules to deploy to the Network Security Group. When not provided, an NSG including only the built-in roles will be deployed.')
param networkSecurityGroupSecurityRules array = []
@description('A /24 to contain the regional firewall, management, and gateway subnet')
@minLength(10)
@maxLength(18)
param hubVnetAddressSpace string
@description('A /26 under the VNet Address Space for the regional Azure Firewall')
@minLength(10)
@maxLength(18)
param azureFirewallSubnetAddressSpace string
@description('A /27 under the VNet Address Space for our regional On-Prem Gateway')
@minLength(10)
@maxLength(18)
param azureGatewaySubnetAddressSpace string
@description('A /27 under the VNet Address Space for regional Azure Bastion')
@minLength(10)
@maxLength(18)
param azureBastionSubnetAddressSpace string
@description('A /16 to contain the cluster')
@minLength(10)
@maxLength(18)
param clusterVnetAddressSpace string
@description('IP ranges authorized to contact the Kubernetes API server. Passing an empty array will result in no IP restrictions. If any are provided, remember to also provide the public IP of the egress Azure Firewall otherwise your nodes will not be able to talk to the API server (e.g. Flux).')
param clusterAuthorizedIPRanges array = []
@description('Key Vault public network access.')
param keyVaultPublicNetworkAccess string
param kubernetesVersion string
@description('Domain name to use for App Gateway and AKS ingress.')
param domainName string
@description('Your cluster will be bootstrapped from this git repo.')
@minLength(9)
param gitOpsBootstrappingRepoHttpsUrl string
@description('You cluster will be bootstrapped from this branch in the identifed git repo.')
@minLength(1)
param gitOpsBootstrappingRepoBranch string
@description('Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster\'s admin group.')
param clusterAdminAadGroupObjectId string
@description('Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC.')
param a0008NamespaceReaderAadGroupObjectId string
@description('Your AKS control plane Cluster API authentication tenant')
param k8sControlPlaneAuthorizationTenantId string = subscription().tenantId
param appGatewayListenerCertificate string
param aksIngressControllerCertificate string
module hub 'rg-hub/hub-default.bicep' = {
name: 'deploy-hub'
params: {
resourceGroupName: hubResourceGroupName
location: location
subnetIpAddressSpace: subnetIpAddressSpace
hubVnetAddressSpace: hubVnetAddressSpace
azureFirewallSubnetAddressSpace: azureFirewallSubnetAddressSpace
azureGatewaySubnetAddressSpace: azureGatewaySubnetAddressSpace
azureBastionSubnetAddressSpace: azureBastionSubnetAddressSpace
networkSecurityGroupSecurityRules: networkSecurityGroupSecurityRules
}
}
module spoke 'rg-spoke/spoke.bicep' = {
name: 'deploy-spoke'
params: {
resourceGroupName: spokeResourceGroupName
clusterVnetAddressSpace: clusterVnetAddressSpace
hubFwResourceId: hub.outputs.hubFwResourceId
hubLaWorkspaceResourceId: hub.outputs.hubLaWorkspaceResourceId
hubVnetResourceId: hub.outputs.hubVnetId
location: location
}
}
module registry 'rg-spoke/acr.bicep' = {
name: 'deploy-registry'
params: {
location: location
targetVnetResourceId: spoke.outputs.clusterVnetResourceId
geoRedundancyLocation: geoRedundancyLocation
resourceGroupName: spokeResourceGroupName
}
}
module clusterprereq 'rg-spoke/clusterprereq.bicep' = {
name: 'deploay-clusterprereq'
params: {
aksIngressControllerCertificate: aksIngressControllerCertificate
appGatewayListenerCertificate: appGatewayListenerCertificate
domainName: domainName
keyVaultPublicNetworkAccess: keyVaultPublicNetworkAccess
location: location
targetVnetResourceId: spoke.outputs.clusterVnetResourceId
vNetResourceGroup: spokeResourceGroupName
resourceGroupName: aksResourceGroupName
}
}
module cluster 'rg-spoke/cluster.bicep' = {
name: 'deploay-cluster'
params: {
a0008NamespaceReaderAadGroupObjectId: a0008NamespaceReaderAadGroupObjectId
clusterAdminAadGroupObjectId: clusterAdminAadGroupObjectId
domainName: domainName
gitOpsBootstrappingRepoBranch: gitOpsBootstrappingRepoBranch
gitOpsBootstrappingRepoHttpsUrl: gitOpsBootstrappingRepoHttpsUrl
kubernetesVersion: kubernetesVersion
location: location
targetVnetResourceId: spoke.outputs.clusterVnetResourceId
vNetResourceGroup: spokeResourceGroupName
resourceGroupName: aksResourceGroupName
clusterAuthorizedIPRanges: clusterAuthorizedIPRanges
k8sControlPlaneAuthorizationTenantId: k8sControlPlaneAuthorizationTenantId
}
dependsOn: [
clusterprereq
]
}
/*** OUTPUTS ***/
output containerRegistryName string = registry.outputs.containerRegistryName
output keyVaultName string = clusterprereq.outputs.keyVaultName
output aksIngressControllerPodManagedIdentityResourceId string = clusterprereq.outputs.aksIngressControllerPodManagedIdentityResourceId
output aksClusterName string = cluster.outputs.aksClusterName
output hubVnetId string = hub.outputs.hubVnetId

Просмотреть файл

@ -0,0 +1,224 @@
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"hubResourceGroupName": {
"value": "rg-enterprise-networking-hubs"
},
"spokeResourceGroupName": {
"value": "rg-enterprise-networking-spokes"
},
"aksResourceGroupName": {
"value": "rg-bu0001a0008"
},
"location": {
"value": "westus2"
},
"geoRedundancyLocation": {
"value": "northeurope"
},
"subnetIpAddressSpace": {
"value": [
"10.240.0.0/16"
]
},
"hubVnetAddressSpace": {
"value": "10.200.0.0/24"
},
"azureFirewallSubnetAddressSpace": {
"value": "10.200.0.0/26"
},
"azureGatewaySubnetAddressSpace": {
"value": "10.200.0.64/27"
},
"azureBastionSubnetAddressSpace": {
"value": "10.200.0.96/27"
},
"clusterVnetAddressSpace": {
"value": "10.240.0.0/16"
},
"domainName": {
"value": "contoso.com"
},
"keyVaultPublicNetworkAccess": {
"value": "Enabled"
},
"kubernetesVersion": {
"value": "1.23.8"
},
"gitOpsBootstrappingRepoHttpsUrl": {
"value": "https://github.com/Azure/aks-baseline-automation"
},
"gitOpsBootstrappingRepoBranch": {
"value": "main"
},
"clusterAdminAadGroupObjectId": {
"value": ""
},
"a0008NamespaceReaderAadGroupObjectId": {
"value": ""
},
"networkSecurityGroupSecurityRules": {
"value": [
{
"name": "AllowWebExperienceInBound",
"properties": {
"description": "Allow our users in. Update this to be as restrictive as possible.",
"protocol": "Tcp",
"sourcePortRange": "*",
"sourceAddressPrefix": "Internet",
"destinationPortRange": "443",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 100,
"direction": "Inbound"
}
},
{
"name": "AllowControlPlaneInBound",
"properties": {
"description": "Service Requirement. Allow control plane access. Regional Tag not yet supported.",
"protocol": "Tcp",
"sourcePortRange": "*",
"sourceAddressPrefix": "GatewayManager",
"destinationPortRange": "443",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 110,
"direction": "Inbound"
}
},
{
"name": "AllowHealthProbesInBound",
"properties": {
"description": "Service Requirement. Allow Health Probes.",
"protocol": "Tcp",
"sourcePortRange": "*",
"sourceAddressPrefix": "AzureLoadBalancer",
"destinationPortRange": "443",
"destinationAddressPrefix": "*",
"access": "Allow",
"priority": 120,
"direction": "Inbound"
}
},
{
"name": "AllowBastionHostToHostInBound",
"properties": {
"description": "Service Requirement. Allow Required Host to Host Communication.",
"protocol": "*",
"sourcePortRange": "*",
"sourceAddressPrefix": "VirtualNetwork",
"destinationPortRanges": [
"8080",
"5701"
],
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 130,
"direction": "Inbound"
}
},
{
"name": "DenyAllInBound",
"properties": {
"protocol": "*",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "*",
"destinationAddressPrefix": "*",
"access": "Deny",
"priority": 1000,
"direction": "Inbound"
}
},
{
"name": "AllowSshToVnetOutBound",
"properties": {
"description": "Allow SSH out to the VNet",
"protocol": "Tcp",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "22",
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 100,
"direction": "Outbound"
}
},
{
"name": "AllowRdpToVnetOutBound",
"properties": {
"protocol": "Tcp",
"description": "Allow RDP out to the VNet",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "3389",
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 110,
"direction": "Outbound"
}
},
{
"name": "AllowControlPlaneOutBound",
"properties": {
"description": "Required for control plane outbound. Regional prefix not yet supported",
"protocol": "Tcp",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "443",
"destinationAddressPrefix": "AzureCloud",
"access": "Allow",
"priority": 120,
"direction": "Outbound"
}
},
{
"name": "AllowBastionHostToHostOutBound",
"properties": {
"description": "Service Requirement. Allow Required Host to Host Communication.",
"protocol": "*",
"sourcePortRange": "*",
"sourceAddressPrefix": "VirtualNetwork",
"destinationPortRanges": [
"8080",
"5701"
],
"destinationAddressPrefix": "VirtualNetwork",
"access": "Allow",
"priority": 130,
"direction": "Outbound"
}
},
{
"name": "AllowBastionCertificateValidationOutBound",
"properties": {
"description": "Service Requirement. Allow Required Session and Certificate Validation.",
"protocol": "*",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "80",
"destinationAddressPrefix": "Internet",
"access": "Allow",
"priority": 140,
"direction": "Outbound"
}
},
{
"name": "DenyAllOutBound",
"properties": {
"protocol": "*",
"sourcePortRange": "*",
"sourceAddressPrefix": "*",
"destinationPortRange": "*",
"destinationAddressPrefix": "*",
"access": "Deny",
"priority": 1000,
"direction": "Outbound"
}
}
]
}
}
}

Просмотреть файл

@ -51,7 +51,7 @@ var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/${clusterNod
var keyVaultName = 'kv-${clusterName}' var keyVaultName = 'kv-${clusterName}'
module rg '../CARML/Microsoft.Resources/resourceGroups/deploy.bicep' = { module rg '../CARML/Microsoft.Resources/resourceGroups/deploy.bicep' = {
name: resourceGroupName name: 'resourceGroup'
params: { params: {
name: resourceGroupName name: resourceGroupName
location: location location: location
@ -72,7 +72,7 @@ module rg '../CARML/Microsoft.Resources/resourceGroups/deploy.bicep' = {
// } // }
module clusterLa '../CARML/Microsoft.OperationalInsights/workspaces/deploy.bicep' = { module clusterLa '../CARML/Microsoft.OperationalInsights/workspaces/deploy.bicep' = {
name: logAnalyticsWorkspaceName name: 'logAnalyticsWorkspace'
params: { params: {
name: logAnalyticsWorkspaceName name: logAnalyticsWorkspaceName
location: location location: location
@ -174,7 +174,7 @@ module podmi_ingress_controller '../CARML/Microsoft.ManagedIdentity/userAssigned
} }
module akvPrivateDnsZones '../CARML/Microsoft.Network/privateDnsZones/deploy.bicep' = { module akvPrivateDnsZones '../CARML/Microsoft.Network/privateDnsZones/deploy.bicep' = {
name: akvPrivateDnsZonesName name: 'akvPrivateDnsZones'
params: { params: {
name: akvPrivateDnsZonesName name: akvPrivateDnsZonesName
location: 'global' location: 'global'
@ -193,7 +193,7 @@ module akvPrivateDnsZones '../CARML/Microsoft.Network/privateDnsZones/deploy.bic
} }
module keyVault '../CARML/Microsoft.KeyVault/vaults/deploy.bicep' = { module keyVault '../CARML/Microsoft.KeyVault/vaults/deploy.bicep' = {
name: keyVaultName name: 'keyVault'
params: { params: {
name: keyVaultName name: keyVaultName
location: location location: location
@ -286,7 +286,7 @@ module backendCert '../CARML/Microsoft.KeyVault/vaults/secrets/deploy.bicep' = {
} }
module wafPolicy '../CARML/Microsoft.Network/applicationGatewayWebApplicationFirewallPolicies/deploy.bicep' = { module wafPolicy '../CARML/Microsoft.Network/applicationGatewayWebApplicationFirewallPolicies/deploy.bicep' = {
name: 'waf-${clusterName}' name: 'waf'
params: { params: {
location: location location: location
name:'waf-${clusterName}' name:'waf-${clusterName}'
@ -311,10 +311,13 @@ module wafPolicy '../CARML/Microsoft.Network/applicationGatewayWebApplicationFir
} }
} }
scope: resourceGroup(resourceGroupName) scope: resourceGroup(resourceGroupName)
dependsOn: [
rg
]
} }
module agw '../CARML/Microsoft.Network/applicationGateways/deploy.bicep' = { module agw '../CARML/Microsoft.Network/applicationGateways/deploy.bicep' = {
name: agwName name: 'agw'
params: { params: {
name: agwName name: agwName
location: location location: location