From 565de09695c6d4e64ebbf7af04251fdd89f4bfbf Mon Sep 17 00:00:00 2001 From: Nate Date: Thu, 30 Apr 2020 10:06:33 -0700 Subject: [PATCH] Updated MSI support (#1399) * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * rolling back terraform version change * Adding aks resource id to output * removing agent_pool_profile which is now considered EOL * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * Adding node_count * Adding msi_enabled var to aks-gitops module * adding system assigned identity outputs * adding system assigned identity outputs * adding system assigned identity outputs * exporting client id through data external script * Adding subscription is * Adding subscription is * removing tenant id output * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet resource id * Adding kubelet resource id * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * removing kubelet identity default * Adding vnet subnet id * version bump * creating dynamic block for sp provision * version bump * fixed aks bug * fixed aks bug * running dos2unix * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * rolling back version change * removing user identity setup and adding node resource group export * reverting flexvol changes * adding nelwine * Adding condition to support aks auto generating sp if sp client id isn't specified * reverting windows profile change * Adding sp terraform variables as optional in aks-gitops module * Adding newline * fixing node group export bug * fixing node group export bug * changing script execution permisssions * key path for gitops rename * update * Fixing SP provisioning bug * removing template for msi Co-authored-by: erikschlegel Co-authored-by: Erik Schlegel --- cluster/azure/aks-gitops/main.tf | 1 + cluster/azure/aks-gitops/outputs.tf | 24 +++++++++++ cluster/azure/aks-gitops/variables.tf | 23 ++++++---- cluster/azure/aks/aks_msi_client_id_query.sh | 4 ++ cluster/azure/aks/main.tf | 38 +++++++++++++++-- cluster/azure/aks/outputs.tf | 24 +++++++++++ cluster/azure/aks/variables.tf | 44 ++++++++++++-------- 7 files changed, 130 insertions(+), 28 deletions(-) create mode 100755 cluster/azure/aks/aks_msi_client_id_query.sh diff --git a/cluster/azure/aks-gitops/main.tf b/cluster/azure/aks-gitops/main.tf index 7a881bf..5777103 100644 --- a/cluster/azure/aks-gitops/main.tf +++ b/cluster/azure/aks-gitops/main.tf @@ -12,6 +12,7 @@ module "aks" { dns_prefix = var.dns_prefix vnet_subnet_id = var.vnet_subnet_id ssh_public_key = var.ssh_public_key + msi_enabled = var.msi_enabled service_principal_id = var.service_principal_id service_principal_secret = var.service_principal_secret service_cidr = var.service_cidr diff --git a/cluster/azure/aks-gitops/outputs.tf b/cluster/azure/aks-gitops/outputs.tf index 671586a..636f029 100644 --- a/cluster/azure/aks-gitops/outputs.tf +++ b/cluster/azure/aks-gitops/outputs.tf @@ -5,3 +5,27 @@ output "kubeconfig_done" { output "aks_flux_kubediff_done" { value = "${module.aks.kubeconfig_done}_${module.flux.flux_done}_${module.kubediff.kubediff_done}" } + +output "aks_resource_id" { + value = module.aks.resource_id +} + +output "msi_client_id" { + value = module.aks.msi_client_id +} + +output "kubelet_client_id" { + value = module.aks.kubelet_client_id +} + +output "kubelet_id" { + value = module.aks.kubelet_id +} + +output "kubelet_resource_id" { + value = module.aks.kubelet_resource_id +} + +output "node_resource_group" { + value = module.aks.node_resource_group +} diff --git a/cluster/azure/aks-gitops/variables.tf b/cluster/azure/aks-gitops/variables.tf index 39c57e0..85475b1 100644 --- a/cluster/azure/aks-gitops/variables.tf +++ b/cluster/azure/aks-gitops/variables.tf @@ -19,6 +19,11 @@ variable "cluster_name" { type = string } +variable "msi_enabled" { + type = bool + default = false +} + variable "dns_prefix" { type = string } @@ -67,14 +72,6 @@ variable "resource_group_name" { type = string } -variable "service_principal_id" { - type = string -} - -variable "service_principal_secret" { - type = string -} - variable "ssh_public_key" { type = string } @@ -83,6 +80,16 @@ variable "vnet_subnet_id" { type = string } +variable "service_principal_id" { + type = string + default = "" +} + +variable "service_principal_secret" { + type = string + default = "" +} + variable "service_cidr" { default = "10.0.0.0/16" description = "Used to assign internal services in the AKS cluster an IP address. This IP address range should be an address space that isn't in use elsewhere in your network environment. This includes any on-premises network ranges if you connect, or plan to connect, your Azure virtual networks using Express Route or a Site-to-Site VPN connections." diff --git a/cluster/azure/aks/aks_msi_client_id_query.sh b/cluster/azure/aks/aks_msi_client_id_query.sh new file mode 100755 index 0000000..0fe9aaf --- /dev/null +++ b/cluster/azure/aks/aks_msi_client_id_query.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +az aks show -n $1 -g $2 --subscription $3 --query "{kubelet_client_id:identityProfile.kubeletidentity.objectId,msi_client_id:identity.principalId,kubelet_id:identityProfile.kubeletidentity.resourceId,kubelet_resource_id:identityProfile.kubeletidentity.resourceId,node_resource_group:nodeResourceGroup}" \ No newline at end of file diff --git a/cluster/azure/aks/main.tf b/cluster/azure/aks/main.tf index 9c9a833..07b3c4f 100644 --- a/cluster/azure/aks/main.tf +++ b/cluster/azure/aks/main.tf @@ -1,7 +1,13 @@ +locals { + msi_identity_type = "SystemAssigned" +} + data "azurerm_resource_group" "cluster" { name = var.resource_group_name } +data "azurerm_subscription" "current" {} + resource "random_id" "workspace" { keepers = { group_name = data.azurerm_resource_group.cluster.name @@ -73,9 +79,15 @@ resource "azurerm_kubernetes_cluster" "cluster" { enabled = true } - service_principal { - client_id = var.service_principal_id - client_secret = var.service_principal_secret + dynamic "service_principal" { + for_each = !var.msi_enabled && var.service_principal_id != "" ? [{ + client_id = var.service_principal_id + client_secret = var.service_principal_secret + }] : [] + content { + client_id = service_principal.value.client_id + client_secret = service_principal.value.client_secret + } } addon_profile { @@ -84,4 +96,24 @@ resource "azurerm_kubernetes_cluster" "cluster" { log_analytics_workspace_id = azurerm_log_analytics_workspace.workspace.id } } + + # This dynamic block enables managed service identity for the cluster + # in the case that the following holds true: + # 1: the msi_enabled input variable is set to true + dynamic "identity" { + for_each = var.msi_enabled ? [local.msi_identity_type] : [] + content { + type = identity.value + } + } +} + +data "external" "msi_object_id" { + depends_on = [azurerm_kubernetes_cluster.cluster] + program = [ + "${path.module}/aks_msi_client_id_query.sh", + var.cluster_name, + data.azurerm_resource_group.cluster.name, + data.azurerm_subscription.current.subscription_id + ] } diff --git a/cluster/azure/aks/outputs.tf b/cluster/azure/aks/outputs.tf index 1f9d8da..c8ccbe6 100644 --- a/cluster/azure/aks/outputs.tf +++ b/cluster/azure/aks/outputs.tf @@ -11,3 +11,27 @@ output "kube_config" { output "kubeconfig_done" { value = join("", local_file.cluster_credentials.*.id) } + +output "resource_id" { + value = azurerm_kubernetes_cluster.cluster.id +} + +output "msi_client_id" { + value = data.external.msi_object_id.result.msi_client_id +} + +output "kubelet_client_id" { + value = data.external.msi_object_id.result.kubelet_client_id +} + +output "kubelet_id" { + value = data.external.msi_object_id.result.kubelet_id +} + +output "node_resource_group" { + value = data.external.msi_object_id.result.node_resource_group +} + +output "kubelet_resource_id" { + value = data.external.msi_object_id.result.kubelet_resource_id +} diff --git a/cluster/azure/aks/variables.tf b/cluster/azure/aks/variables.tf index 70291ae..9dacc89 100644 --- a/cluster/azure/aks/variables.tf +++ b/cluster/azure/aks/variables.tf @@ -2,21 +2,40 @@ variable "resource_group_name" { type = string } -variable "cluster_name" { - type = string - default = "bedrockaks" -} - variable "dns_prefix" { type = string } +variable "kubernetes_version" { + type = string +} + +variable "ssh_public_key" { + type = string +} + +variable "vnet_subnet_id" { + type = string +} + variable "service_principal_id" { type = string + default = "" } variable "service_principal_secret" { type = string + default = "" +} + +variable "msi_enabled" { + type = bool + default = false +} + +variable "cluster_name" { + type = string + default = "bedrockaks" } variable "agent_vm_count" { @@ -29,28 +48,16 @@ variable "agent_vm_size" { default = "Standard_D2s_v3" } -variable "kubernetes_version" { - type = string -} - variable "admin_user" { type = string default = "k8sadmin" } -variable "ssh_public_key" { - type = string -} - variable "output_directory" { type = string default = "./output" } -variable "vnet_subnet_id" { - type = string -} - variable "enable_virtual_node_addon" { type = string default = "false" @@ -81,16 +88,19 @@ variable "dns_ip" { } variable "docker_cidr" { + type = string default = "172.17.0.1/16" description = "IP address (in CIDR notation) used as the Docker bridge IP address on nodes. Default of 172.17.0.1/16." } variable "network_plugin" { default = "azure" + type = string description = "Network plugin used by AKS. Either azure or kubenet." } variable "network_policy" { default = "azure" + type = string description = "Network policy to be used with Azure CNI. Either azure or calico." }