From d4b2b49b57fd3bd9d12ad70a587b0e057e5d0fc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20Pei=C3=9Fker?= <30857628+JPEasier@users.noreply.github.com> Date: Tue, 18 Apr 2023 09:04:31 +0200 Subject: [PATCH] Add top level biceps file (#117) * add wrapper bicep file * update pipeline * fix nsg typo * update dependency * update dependency * default kube version * update resourcegroup name * update pipeline * update kv script * moving hardcoded values into parameter file * remove certificates from param file * adding parameter file into pipeline * remove default values --------- Co-authored-by: Ayobami Ayodeji Co-authored-by: Rainer Halanek Co-authored-by: Rainer Halanek <61878316+rahalan@users.noreply.github.com> --- .github/workflows/IaC-bicep-AKS.yml | 70 ++------ IaC/bicep/main.bicep | 163 ++++++++++++++++++ IaC/bicep/main.parameters.json | 224 +++++++++++++++++++++++++ IaC/bicep/rg-spoke/clusterprereq.bicep | 15 +- 4 files changed, 407 insertions(+), 65 deletions(-) create mode 100644 IaC/bicep/main.bicep create mode 100644 IaC/bicep/main.parameters.json diff --git a/.github/workflows/IaC-bicep-AKS.yml b/.github/workflows/IaC-bicep-AKS.yml index 9b41143..7fdfe5b 100644 --- a/.github/workflows/IaC-bicep-AKS.yml +++ b/.github/workflows/IaC-bicep-AKS.yml @@ -116,82 +116,34 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - name: "Deploy Hub" - id: hub + - name: "Deploy AKS Landingzone" + id: akslz uses: azure/arm-deploy@v1 with: subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} region: ${{ github.event.inputs.REGION }} scope: subscription - template: ./IaC/bicep/rg-hub/hub-default.bicep - parameters: ./IaC/bicep/rg-hub/hub-default.parameters.json resourceGroupName=rg-enterprise-networking-hubs-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} + template: ./IaC/bicep/main.bicep + parameters: ./IaC/bicep/main.parameters.json clusterAdminAadGroupObjectId=${{ github.event.inputs.clusterAdminAadGroupObjectId }} a0008NamespaceReaderAadGroupObjectId=${{ github.event.inputs.a0008NamespaceReaderAadGroupObjectId }} appGatewayListenerCertificate=${{ env.APP_GATEWAY_LISTENER_CERTIFICATE }} aksIngressControllerCertificate=${{ env.AKS_INGRESS_CONTROLLER_CERTIFICATE }} failOnStdErr: false - deploymentName: carml-hub-${{ github.event.inputs.REGION }} - - - name: "Deploy Spoke" - id: spoke - uses: azure/arm-deploy@v1 - with: - subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} - region: ${{ github.event.inputs.REGION }} - scope: subscription - template: ./IaC/bicep/rg-spoke/spoke.bicep - parameters: ./IaC/bicep/rg-spoke/spoke.parameters.json hubVnetResourceId=${{ steps.hub.outputs.hubVnetId }} hubLaWorkspaceResourceId=${{ steps.hub.outputs.hubLaWorkspaceResourceId }} hubFwResourceId=${{ steps.hub.outputs.hubFwResourceId }} resourceGroupName=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} - failOnStdErr: false - deploymentName: carml-spoke-${{ github.event.inputs.REGION }} - - - name: "Deploy Registry" - id: registry - uses: azure/arm-deploy@v1 - with: - subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} - region: ${{ github.event.inputs.REGION }} - scope: subscription - template: ./IaC/bicep/rg-spoke/acr.bicep - parameters: ./IaC/bicep/rg-spoke/acr.parameters.json targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }} - failOnStdErr: false - deploymentName: carml-registry-${{ github.event.inputs.REGION }} + deploymentName: carml-aks-landingzone-${{ github.event.inputs.REGION }} # Import core images hosted in public container registries to be used during bootstrapping - name: "Import Images into ACR for flux" id: image_import run: | - az acr import --source docker.io/weaveworks/kured:1.10.1 -n ${{ steps.registry.outputs.containerRegistryName }} --force - az acr import --source docker.io/library/traefik:v2.8.1 -n ${{ steps.registry.outputs.containerRegistryName }} --force - - - name: "Deploy Cluster prerequisites" - id: cluster_prereq - uses: azure/arm-deploy@v1 - with: - subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} - region: ${{ github.event.inputs.REGION }} - scope: subscription - template: ./IaC/bicep/rg-spoke/clusterprereq.bicep - parameters: ./IaC/bicep/rg-spoke/clusterprereq.parameters.json appGatewayListenerCertificate=${{ env.APP_GATEWAY_LISTENER_CERTIFICATE }} aksIngressControllerCertificate=${{ env.AKS_INGRESS_CONTROLLER_CERTIFICATE }} targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} vNetResourceGroup=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }} - failOnStdErr: false - deploymentName: carml-cluster-${{ github.event.inputs.REGION }} - - - name: "Deploy Cluster" - id: cluster - uses: azure/arm-deploy@v1 - with: - subscriptionId: ${{ secrets.SUBSCRIPTION_ID }} - region: ${{ github.event.inputs.REGION }} - scope: subscription - template: ./IaC/bicep/rg-spoke/cluster.bicep - parameters: ./IaC/bicep/rg-spoke/cluster.parameters.json targetVnetResourceId=${{ steps.spoke.outputs.clusterVnetResourceId }} clusterAdminAadGroupObjectId=${{ github.event.inputs.clusterAdminAadGroupObjectId }} a0008NamespaceReaderAadGroupObjectId=${{ github.event.inputs.a0008NamespaceReaderAadGroupObjectId }} vNetResourceGroup=rg-enterprise-networking-spokes-${{ github.event.inputs.REGION }} location=${{ github.event.inputs.REGION }} resourceGroupName=rg-BU0001A0008-${{ github.event.inputs.REGION }} gitOpsBootstrappingRepoBranch=${{ github.ref_name }} - failOnStdErr: false - deploymentName: carml-cluster-${{ github.event.inputs.REGION }} + az acr import --source docker.io/weaveworks/kured:1.10.1 -n ${{ steps.akslz.outputs.containerRegistryName }} --force + az acr import --source docker.io/library/traefik:v2.8.1 -n ${{ steps.akslz.outputs.containerRegistryName }} --force # Temporary workaround until we figure out why steps.cluster.outputs.keyVaultName is blank - name: "Get KeyVault Name" id: akv_name run: | - export AKV_NAME=$(az deployment group list -g rg-BU0001A0008-${{ github.event.inputs.REGION }} -o table| grep 'kv-aks'|awk '{print $1}') + export AKV_NAME=$(az keyvault list -g rg-bu0001a0008 -o table | grep "kv-aks" | awk '{print $2}') echo "AKV_NAME=${AKV_NAME}" >> $GITHUB_ENV - echo "AKV Name from bicep output is ${{ steps.cluster.outputs.keyVaultName }}" - echo "aksIngressControllerPodManagedIdentityResourceId from bicep output is ${{ steps.cluster.outputs.aksIngressControllerPodManagedIdentityResourceId }}" - echo "To prove that this should work: hubVnetId is ${{ steps.hub.outputs.hubVnetId }}" + echo "AKV Name from bicep output is ${{ steps.akslz.outputs.keyVaultName }}" + echo "aksIngressControllerPodManagedIdentityResourceId from bicep output is ${{ steps.akslz.outputs.aksIngressControllerPodManagedIdentityResourceId }}" + echo "To prove that this should work: hubVnetId is ${{ steps.akslz.outputs.hubVnetId }}" # Re-authentication is required for the commands that follow - name: Azure Login diff --git a/IaC/bicep/main.bicep b/IaC/bicep/main.bicep new file mode 100644 index 0000000..db66a7a --- /dev/null +++ b/IaC/bicep/main.bicep @@ -0,0 +1,163 @@ +targetScope = 'subscription' + +@description('Name of the hub resource group') +param hubResourceGroupName string + +@description('Name of the spoke resource group') +param spokeResourceGroupName string + +@description('Name of the AKS resource group') +param aksResourceGroupName string + +@description('AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +param location string + +@description('For Azure resources that support native geo-redunancy, provide the location the redundant service will have its secondary. Should be different than the location parameter and ideally should be a paired region - https://learn.microsoft.com/azure/best-practices-availability-paired-regions. This region does not need to support availability zones.') +param geoRedundancyLocation string + +@description('Subnet address prefixes for all AKS clusters nodepools in all attached spokes to allow necessary outbound traffic through the firewall.') +@minLength(1) +param subnetIpAddressSpace array + +@description('Optional. Array of Security Rules to deploy to the Network Security Group. When not provided, an NSG including only the built-in roles will be deployed.') +param networkSecurityGroupSecurityRules array = [] + +@description('A /24 to contain the regional firewall, management, and gateway subnet') +@minLength(10) +@maxLength(18) +param hubVnetAddressSpace string + +@description('A /26 under the VNet Address Space for the regional Azure Firewall') +@minLength(10) +@maxLength(18) +param azureFirewallSubnetAddressSpace string + +@description('A /27 under the VNet Address Space for our regional On-Prem Gateway') +@minLength(10) +@maxLength(18) +param azureGatewaySubnetAddressSpace string + +@description('A /27 under the VNet Address Space for regional Azure Bastion') +@minLength(10) +@maxLength(18) +param azureBastionSubnetAddressSpace string + +@description('A /16 to contain the cluster') +@minLength(10) +@maxLength(18) +param clusterVnetAddressSpace string + +@description('IP ranges authorized to contact the Kubernetes API server. Passing an empty array will result in no IP restrictions. If any are provided, remember to also provide the public IP of the egress Azure Firewall otherwise your nodes will not be able to talk to the API server (e.g. Flux).') +param clusterAuthorizedIPRanges array = [] + +@description('Key Vault public network access.') +param keyVaultPublicNetworkAccess string + +param kubernetesVersion string + +@description('Domain name to use for App Gateway and AKS ingress.') +param domainName string + +@description('Your cluster will be bootstrapped from this git repo.') +@minLength(9) +param gitOpsBootstrappingRepoHttpsUrl string + +@description('You cluster will be bootstrapped from this branch in the identifed git repo.') +@minLength(1) +param gitOpsBootstrappingRepoBranch string + +@description('Azure AD Group in the identified tenant that will be granted the highly privileged cluster-admin role. If Azure RBAC is used, then this group will get a role assignment to Azure RBAC, else it will be assigned directly to the cluster\'s admin group.') +param clusterAdminAadGroupObjectId string + +@description('Azure AD Group in the identified tenant that will be granted the read only privileges in the a0008 namespace that exists in the cluster. This is only used when Azure RBAC is used for Kubernetes RBAC.') +param a0008NamespaceReaderAadGroupObjectId string + +@description('Your AKS control plane Cluster API authentication tenant') +param k8sControlPlaneAuthorizationTenantId string = subscription().tenantId + +param appGatewayListenerCertificate string + +param aksIngressControllerCertificate string + +module hub 'rg-hub/hub-default.bicep' = { + name: 'deploy-hub' + params: { + resourceGroupName: hubResourceGroupName + location: location + subnetIpAddressSpace: subnetIpAddressSpace + hubVnetAddressSpace: hubVnetAddressSpace + azureFirewallSubnetAddressSpace: azureFirewallSubnetAddressSpace + azureGatewaySubnetAddressSpace: azureGatewaySubnetAddressSpace + azureBastionSubnetAddressSpace: azureBastionSubnetAddressSpace + networkSecurityGroupSecurityRules: networkSecurityGroupSecurityRules + } +} + +module spoke 'rg-spoke/spoke.bicep' = { + name: 'deploy-spoke' + params: { + resourceGroupName: spokeResourceGroupName + clusterVnetAddressSpace: clusterVnetAddressSpace + hubFwResourceId: hub.outputs.hubFwResourceId + hubLaWorkspaceResourceId: hub.outputs.hubLaWorkspaceResourceId + hubVnetResourceId: hub.outputs.hubVnetId + location: location + } +} + +module registry 'rg-spoke/acr.bicep' = { + name: 'deploy-registry' + params: { + location: location + targetVnetResourceId: spoke.outputs.clusterVnetResourceId + geoRedundancyLocation: geoRedundancyLocation + resourceGroupName: spokeResourceGroupName + } +} + +module clusterprereq 'rg-spoke/clusterprereq.bicep' = { + name: 'deploay-clusterprereq' + params: { + aksIngressControllerCertificate: aksIngressControllerCertificate + appGatewayListenerCertificate: appGatewayListenerCertificate + domainName: domainName + keyVaultPublicNetworkAccess: keyVaultPublicNetworkAccess + location: location + targetVnetResourceId: spoke.outputs.clusterVnetResourceId + vNetResourceGroup: spokeResourceGroupName + resourceGroupName: aksResourceGroupName + } +} + +module cluster 'rg-spoke/cluster.bicep' = { + name: 'deploay-cluster' + params: { + a0008NamespaceReaderAadGroupObjectId: a0008NamespaceReaderAadGroupObjectId + clusterAdminAadGroupObjectId: clusterAdminAadGroupObjectId + domainName: domainName + gitOpsBootstrappingRepoBranch: gitOpsBootstrappingRepoBranch + gitOpsBootstrappingRepoHttpsUrl: gitOpsBootstrappingRepoHttpsUrl + kubernetesVersion: kubernetesVersion + location: location + targetVnetResourceId: spoke.outputs.clusterVnetResourceId + vNetResourceGroup: spokeResourceGroupName + resourceGroupName: aksResourceGroupName + clusterAuthorizedIPRanges: clusterAuthorizedIPRanges + k8sControlPlaneAuthorizationTenantId: k8sControlPlaneAuthorizationTenantId + } + dependsOn: [ + clusterprereq + ] +} + +/*** OUTPUTS ***/ + +output containerRegistryName string = registry.outputs.containerRegistryName + +output keyVaultName string = clusterprereq.outputs.keyVaultName + +output aksIngressControllerPodManagedIdentityResourceId string = clusterprereq.outputs.aksIngressControllerPodManagedIdentityResourceId + +output aksClusterName string = cluster.outputs.aksClusterName + +output hubVnetId string = hub.outputs.hubVnetId diff --git a/IaC/bicep/main.parameters.json b/IaC/bicep/main.parameters.json new file mode 100644 index 0000000..7b1c2d5 --- /dev/null +++ b/IaC/bicep/main.parameters.json @@ -0,0 +1,224 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "hubResourceGroupName": { + "value": "rg-enterprise-networking-hubs" + }, + "spokeResourceGroupName": { + "value": "rg-enterprise-networking-spokes" + }, + "aksResourceGroupName": { + "value": "rg-bu0001a0008" + }, + "location": { + "value": "westus2" + }, + "geoRedundancyLocation": { + "value": "northeurope" + }, + "subnetIpAddressSpace": { + "value": [ + "10.240.0.0/16" + ] + }, + "hubVnetAddressSpace": { + "value": "10.200.0.0/24" + }, + "azureFirewallSubnetAddressSpace": { + "value": "10.200.0.0/26" + }, + "azureGatewaySubnetAddressSpace": { + "value": "10.200.0.64/27" + }, + "azureBastionSubnetAddressSpace": { + "value": "10.200.0.96/27" + }, + "clusterVnetAddressSpace": { + "value": "10.240.0.0/16" + }, + "domainName": { + "value": "contoso.com" + }, + "keyVaultPublicNetworkAccess": { + "value": "Enabled" + }, + "kubernetesVersion": { + "value": "1.23.8" + }, + "gitOpsBootstrappingRepoHttpsUrl": { + "value": "https://github.com/Azure/aks-baseline-automation" + }, + "gitOpsBootstrappingRepoBranch": { + "value": "main" + }, + "clusterAdminAadGroupObjectId": { + "value": "" + }, + "a0008NamespaceReaderAadGroupObjectId": { + "value": "" + }, + "networkSecurityGroupSecurityRules": { + "value": [ + { + "name": "AllowWebExperienceInBound", + "properties": { + "description": "Allow our users in. Update this to be as restrictive as possible.", + "protocol": "Tcp", + "sourcePortRange": "*", + "sourceAddressPrefix": "Internet", + "destinationPortRange": "443", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 100, + "direction": "Inbound" + } + }, + { + "name": "AllowControlPlaneInBound", + "properties": { + "description": "Service Requirement. Allow control plane access. Regional Tag not yet supported.", + "protocol": "Tcp", + "sourcePortRange": "*", + "sourceAddressPrefix": "GatewayManager", + "destinationPortRange": "443", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 110, + "direction": "Inbound" + } + }, + { + "name": "AllowHealthProbesInBound", + "properties": { + "description": "Service Requirement. Allow Health Probes.", + "protocol": "Tcp", + "sourcePortRange": "*", + "sourceAddressPrefix": "AzureLoadBalancer", + "destinationPortRange": "443", + "destinationAddressPrefix": "*", + "access": "Allow", + "priority": 120, + "direction": "Inbound" + } + }, + { + "name": "AllowBastionHostToHostInBound", + "properties": { + "description": "Service Requirement. Allow Required Host to Host Communication.", + "protocol": "*", + "sourcePortRange": "*", + "sourceAddressPrefix": "VirtualNetwork", + "destinationPortRanges": [ + "8080", + "5701" + ], + "destinationAddressPrefix": "VirtualNetwork", + "access": "Allow", + "priority": 130, + "direction": "Inbound" + } + }, + { + "name": "DenyAllInBound", + "properties": { + "protocol": "*", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "*", + "destinationAddressPrefix": "*", + "access": "Deny", + "priority": 1000, + "direction": "Inbound" + } + }, + { + "name": "AllowSshToVnetOutBound", + "properties": { + "description": "Allow SSH out to the VNet", + "protocol": "Tcp", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "22", + "destinationAddressPrefix": "VirtualNetwork", + "access": "Allow", + "priority": 100, + "direction": "Outbound" + } + }, + { + "name": "AllowRdpToVnetOutBound", + "properties": { + "protocol": "Tcp", + "description": "Allow RDP out to the VNet", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "3389", + "destinationAddressPrefix": "VirtualNetwork", + "access": "Allow", + "priority": 110, + "direction": "Outbound" + } + }, + { + "name": "AllowControlPlaneOutBound", + "properties": { + "description": "Required for control plane outbound. Regional prefix not yet supported", + "protocol": "Tcp", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "443", + "destinationAddressPrefix": "AzureCloud", + "access": "Allow", + "priority": 120, + "direction": "Outbound" + } + }, + { + "name": "AllowBastionHostToHostOutBound", + "properties": { + "description": "Service Requirement. Allow Required Host to Host Communication.", + "protocol": "*", + "sourcePortRange": "*", + "sourceAddressPrefix": "VirtualNetwork", + "destinationPortRanges": [ + "8080", + "5701" + ], + "destinationAddressPrefix": "VirtualNetwork", + "access": "Allow", + "priority": 130, + "direction": "Outbound" + } + }, + { + "name": "AllowBastionCertificateValidationOutBound", + "properties": { + "description": "Service Requirement. Allow Required Session and Certificate Validation.", + "protocol": "*", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "80", + "destinationAddressPrefix": "Internet", + "access": "Allow", + "priority": 140, + "direction": "Outbound" + } + }, + { + "name": "DenyAllOutBound", + "properties": { + "protocol": "*", + "sourcePortRange": "*", + "sourceAddressPrefix": "*", + "destinationPortRange": "*", + "destinationAddressPrefix": "*", + "access": "Deny", + "priority": 1000, + "direction": "Outbound" + } + } + ] + } + } +} \ No newline at end of file diff --git a/IaC/bicep/rg-spoke/clusterprereq.bicep b/IaC/bicep/rg-spoke/clusterprereq.bicep index f1cc3fc..ce8ab34 100644 --- a/IaC/bicep/rg-spoke/clusterprereq.bicep +++ b/IaC/bicep/rg-spoke/clusterprereq.bicep @@ -51,7 +51,7 @@ var vnetNodePoolSubnetResourceId = '${targetVnetResourceId}/subnets/${clusterNod var keyVaultName = 'kv-${clusterName}' module rg '../CARML/Microsoft.Resources/resourceGroups/deploy.bicep' = { - name: resourceGroupName + name: 'resourceGroup' params: { name: resourceGroupName location: location @@ -72,7 +72,7 @@ module rg '../CARML/Microsoft.Resources/resourceGroups/deploy.bicep' = { // } module clusterLa '../CARML/Microsoft.OperationalInsights/workspaces/deploy.bicep' = { - name: logAnalyticsWorkspaceName + name: 'logAnalyticsWorkspace' params: { name: logAnalyticsWorkspaceName location: location @@ -174,7 +174,7 @@ module podmi_ingress_controller '../CARML/Microsoft.ManagedIdentity/userAssigned } module akvPrivateDnsZones '../CARML/Microsoft.Network/privateDnsZones/deploy.bicep' = { - name: akvPrivateDnsZonesName + name: 'akvPrivateDnsZones' params: { name: akvPrivateDnsZonesName location: 'global' @@ -193,7 +193,7 @@ module akvPrivateDnsZones '../CARML/Microsoft.Network/privateDnsZones/deploy.bic } module keyVault '../CARML/Microsoft.KeyVault/vaults/deploy.bicep' = { - name: keyVaultName + name: 'keyVault' params: { name: keyVaultName location: location @@ -286,7 +286,7 @@ module backendCert '../CARML/Microsoft.KeyVault/vaults/secrets/deploy.bicep' = { } module wafPolicy '../CARML/Microsoft.Network/applicationGatewayWebApplicationFirewallPolicies/deploy.bicep' = { - name: 'waf-${clusterName}' + name: 'waf' params: { location: location name:'waf-${clusterName}' @@ -311,10 +311,13 @@ module wafPolicy '../CARML/Microsoft.Network/applicationGatewayWebApplicationFir } } scope: resourceGroup(resourceGroupName) + dependsOn: [ + rg + ] } module agw '../CARML/Microsoft.Network/applicationGateways/deploy.bicep' = { - name: agwName + name: 'agw' params: { name: agwName location: location