Updated MSI support (#1399)

* Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * Adding msi integration * rolling back terraform version change * Adding aks resource id to output * removing agent_pool_profile which is now considered EOL * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * removing deprecated vault_name property * Adding node_count * Adding msi_enabled var to aks-gitops module * adding system assigned identity outputs * adding system assigned identity outputs * adding system assigned identity outputs * exporting client id through data external script * Adding subscription is * Adding subscription is * removing tenant id output * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet identity * Adding kubelet resource id * Adding kubelet resource id * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * refactoring aks mod to create use assigned identity * removing kubelet identity default * Adding vnet subnet id * version bump * creating dynamic block for sp provision * version bump * fixed aks bug * fixed aks bug * running dos2unix * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * adding agent pool resource id to output * rolling back version change * removing user identity setup and adding node resource group export * reverting flexvol changes * adding nelwine * Adding condition to support aks auto generating sp if sp client id isn't specified * reverting windows profile change * Adding sp terraform variables as optional in aks-gitops module * Adding newline * fixing node group export bug * fixing node group export bug * changing script execution permisssions * key path for gitops rename * update * Fixing SP provisioning bug * removing template for msi Co-authored-by: erikschlegel <erik.schlegel@gmail.com> Co-authored-by: Erik Schlegel <erisch@microsoft.com>
2020-04-30 10:06:33 -07:00 · 2020-04-30 10:06:33 -07:00 · 565de09695
--- a/cluster/azure/aks-gitops/main.tf
+++ b/cluster/azure/aks-gitops/main.tf
@ -12,6 +12,7 @@ module "aks" {
  dns_prefix               = var.dns_prefix
  vnet_subnet_id           = var.vnet_subnet_id
  ssh_public_key           = var.ssh_public_key
+  msi_enabled              = var.msi_enabled
  service_principal_id     = var.service_principal_id
  service_principal_secret = var.service_principal_secret
  service_cidr             = var.service_cidr
--- a/cluster/azure/aks-gitops/outputs.tf
+++ b/cluster/azure/aks-gitops/outputs.tf
@ -5,3 +5,27 @@ output "kubeconfig_done" {
 output "aks_flux_kubediff_done" {
  value = "${module.aks.kubeconfig_done}_${module.flux.flux_done}_${module.kubediff.kubediff_done}"
 }
+
+output "aks_resource_id" {
+  value = module.aks.resource_id
+}
+
+output "msi_client_id" {
+  value = module.aks.msi_client_id
+}
+
+output "kubelet_client_id" {
+  value = module.aks.kubelet_client_id
+}
+
+output "kubelet_id" {
+  value = module.aks.kubelet_id
+}
+
+output "kubelet_resource_id" {
+  value = module.aks.kubelet_resource_id
+}
+
+output "node_resource_group" {
+  value = module.aks.node_resource_group
+}
--- a/cluster/azure/aks-gitops/variables.tf
+++ b/cluster/azure/aks-gitops/variables.tf
@ -19,6 +19,11 @@ variable "cluster_name" {
  type = string
 }

+variable "msi_enabled" {
+  type = bool
+  default = false
+}
+
 variable "dns_prefix" {
  type = string
 }
@ -67,14 +72,6 @@ variable "resource_group_name" {
  type = string
 }

-variable "service_principal_id" {
-  type = string
-}
-
-variable "service_principal_secret" {
-  type = string
-}
-
 variable "ssh_public_key" {
  type = string
 }
@ -83,6 +80,16 @@ variable "vnet_subnet_id" {
  type = string
 }

+variable "service_principal_id" {
+  type = string
+  default = ""
+}
+
+variable "service_principal_secret" {
+  type = string
+  default = ""
+}
+
 variable "service_cidr" {
  default     = "10.0.0.0/16"
  description = "Used to assign internal services in the AKS cluster an IP address. This IP address range should be an address space that isn't in use elsewhere in your network environment. This includes any on-premises network ranges if you connect, or plan to connect, your Azure virtual networks using Express Route or a Site-to-Site VPN connections."
--- a/cluster/azure/aks/aks_msi_client_id_query.sh
+++ b/cluster/azure/aks/aks_msi_client_id_query.sh
@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+az aks show -n $1 -g $2 --subscription $3 --query "{kubelet_client_id:identityProfile.kubeletidentity.objectId,msi_client_id:identity.principalId,kubelet_id:identityProfile.kubeletidentity.resourceId,kubelet_resource_id:identityProfile.kubeletidentity.resourceId,node_resource_group:nodeResourceGroup}"
--- a/cluster/azure/aks/main.tf
+++ b/cluster/azure/aks/main.tf
@ -1,7 +1,13 @@
+locals {
+  msi_identity_type = "SystemAssigned"
+}
+
 data "azurerm_resource_group" "cluster" {
  name = var.resource_group_name
 }

+data "azurerm_subscription" "current" {}
+
 resource "random_id" "workspace" {
  keepers = {
    group_name = data.azurerm_resource_group.cluster.name
@ -73,9 +79,15 @@ resource "azurerm_kubernetes_cluster" "cluster" {
    enabled = true
  }

-  service_principal {
-    client_id     = var.service_principal_id
-    client_secret = var.service_principal_secret
+  dynamic "service_principal" {
+    for_each = !var.msi_enabled && var.service_principal_id != "" ? [{
+      client_id     = var.service_principal_id
+      client_secret = var.service_principal_secret
+    }] : []
+    content {
+      client_id     = service_principal.value.client_id
+      client_secret = service_principal.value.client_secret
+    }
  }

  addon_profile {
@ -84,4 +96,24 @@ resource "azurerm_kubernetes_cluster" "cluster" {
      log_analytics_workspace_id = azurerm_log_analytics_workspace.workspace.id
    }
  }
+
+  # This dynamic block enables managed service identity for the cluster
+  # in the case that the following holds true:
+  #   1: the msi_enabled input variable is set to true
+  dynamic "identity" {
+    for_each = var.msi_enabled ? [local.msi_identity_type] : []
+    content {
+      type = identity.value
+    }
+  }
+}
+
+data "external" "msi_object_id" {
+  depends_on = [azurerm_kubernetes_cluster.cluster]
+  program = [
+    "${path.module}/aks_msi_client_id_query.sh",
+    var.cluster_name,
+    data.azurerm_resource_group.cluster.name,
+    data.azurerm_subscription.current.subscription_id
+  ]
 }
--- a/cluster/azure/aks/outputs.tf
+++ b/cluster/azure/aks/outputs.tf
@ -11,3 +11,27 @@ output "kube_config" {
 output "kubeconfig_done" {
  value = join("", local_file.cluster_credentials.*.id)
 }
+
+output "resource_id" {
+  value = azurerm_kubernetes_cluster.cluster.id
+}
+
+output "msi_client_id" {
+  value = data.external.msi_object_id.result.msi_client_id
+}
+
+output "kubelet_client_id" {
+  value = data.external.msi_object_id.result.kubelet_client_id
+}
+
+output "kubelet_id" {
+  value = data.external.msi_object_id.result.kubelet_id
+}
+
+output "node_resource_group" {
+  value = data.external.msi_object_id.result.node_resource_group
+}
+
+output "kubelet_resource_id" {
+  value = data.external.msi_object_id.result.kubelet_resource_id
+}
--- a/cluster/azure/aks/variables.tf
+++ b/cluster/azure/aks/variables.tf
@ -2,21 +2,40 @@ variable "resource_group_name" {
  type = string
 }

-variable "cluster_name" {
-  type    = string
-  default = "bedrockaks"
-}
-
 variable "dns_prefix" {
  type = string
 }

+variable "kubernetes_version" {
+  type = string
+}
+
+variable "ssh_public_key" {
+  type = string
+}
+
+variable "vnet_subnet_id" {
+  type = string
+}
+
 variable "service_principal_id" {
  type = string
+  default = ""
 }

 variable "service_principal_secret" {
  type = string
+  default = ""
+}
+
+variable "msi_enabled" {
+  type = bool
+  default = false
+}
+
+variable "cluster_name" {
+  type    = string
+  default = "bedrockaks"
 }

 variable "agent_vm_count" {
@ -29,28 +48,16 @@ variable "agent_vm_size" {
  default = "Standard_D2s_v3"
 }

-variable "kubernetes_version" {
-  type = string
-}
-
 variable "admin_user" {
  type    = string
  default = "k8sadmin"
 }

-variable "ssh_public_key" {
-  type = string
-}
-
 variable "output_directory" {
  type    = string
  default = "./output"
 }

-variable "vnet_subnet_id" {
-  type = string
-}
-
 variable "enable_virtual_node_addon" {
  type    = string
  default = "false"
@ -81,16 +88,19 @@ variable "dns_ip" {
 }

 variable "docker_cidr" {
+  type        = string
  default     = "172.17.0.1/16"
  description = "IP address (in CIDR notation) used as the Docker bridge IP address on nodes. Default of 172.17.0.1/16."
 }

 variable "network_plugin" {
  default     = "azure"
+  type        = string
  description = "Network plugin used by AKS. Either azure or kubenet."
 }
 variable "network_policy" {
  default     = "azure"
+  type        = string
  description = "Network policy to be used with Azure CNI. Either azure or calico."
 }