Update aks module to output identities (#1442)

* update aks module to output identities

* formatting tests and updating kubernetes version

* fix outputs in aks module

cluster/azure/aks-gitops/outputs.tf

@@ -10,22 +10,18 @@ output "aks_resource_id" {
   value = module.aks.resource_id
 }
 
-output "msi_client_id" {
-  value = module.aks.msi_client_id
-}
-
-output "kubelet_client_id" {
-  value = module.aks.kubelet_client_id
-}
-
-output "kubelet_id" {
-  value = module.aks.kubelet_id
-}
-
-output "kubelet_resource_id" {
-  value = module.aks.kubelet_resource_id
-}
-
 output "node_resource_group" {
   value = module.aks.node_resource_group
 }
+
+output "kubelet_identity" {
+  value = module.aks.kubelet_identity
+}
+
+output "system_identity" {
+  value = module.aks.system_identity
+}
+
+output "oms_agent_identity" {
+  value = module.aks.oms_agent_identity
+}

cluster/azure/aks/aks_msi_client_id_query.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-az aks show -n $1 -g $2 --subscription $3 --query "{kubelet_client_id:identityProfile.kubeletidentity.objectId,msi_client_id:identity.principalId,kubelet_id:identityProfile.kubeletidentity.resourceId,kubelet_resource_id:identityProfile.kubeletidentity.resourceId,node_resource_group:nodeResourceGroup}"

cluster/azure/aks/main.tf

@@ -82,7 +82,7 @@ resource "azurerm_kubernetes_cluster" "cluster" {
   }
 
   dynamic "service_principal" {
-    for_each = !var.msi_enabled && var.service_principal_id != "" ? [{
+    for_each = ! var.msi_enabled && var.service_principal_id != "" ? [{
       client_id     = var.service_principal_id
       client_secret = var.service_principal_secret
     }] : []
@@ -117,13 +117,3 @@ resource "azurerm_kubernetes_cluster" "cluster" {
 
   tags = var.tags
 }
-
-data "external" "msi_object_id" {
-  depends_on = [azurerm_kubernetes_cluster.cluster]
-  program = [
-    "${path.module}/aks_msi_client_id_query.sh",
-    var.cluster_name,
-    data.azurerm_resource_group.cluster.name,
-    data.azurerm_subscription.current.subscription_id
-  ]
-}

cluster/azure/aks/outputs.tf

@@ -16,22 +16,18 @@ output "resource_id" {
   value = azurerm_kubernetes_cluster.cluster.id
 }
 
-output "msi_client_id" {
-  value = data.external.msi_object_id.result.msi_client_id
-}
-
-output "kubelet_client_id" {
-  value = data.external.msi_object_id.result.kubelet_client_id
-}
-
-output "kubelet_id" {
-  value = data.external.msi_object_id.result.kubelet_id
-}
-
 output "node_resource_group" {
-  value = data.external.msi_object_id.result.node_resource_group
+  value = azurerm_kubernetes_cluster.cluster.node_resource_group
 }
 
-output "kubelet_resource_id" {
-  value = data.external.msi_object_id.result.kubelet_resource_id
+output "kubelet_identity" {
+  value = length(azurerm_kubernetes_cluster.cluster.kubelet_identity) > 0 ? azurerm_kubernetes_cluster.cluster.kubelet_identity[0] : null
+}
+
+output "system_identity" {
+  value = length(azurerm_kubernetes_cluster.cluster.identity) > 0 ? azurerm_kubernetes_cluster.cluster.identity[0] : null
+}
+
+output "oms_agent_identity" {
+  value = var.oms_agent_enabled ? azurerm_kubernetes_cluster.cluster.addon_profile[0].oms_agent[0].oms_agent_identity[0] : null
 }

cluster/azure/aks/variables.tf

@@ -19,17 +19,17 @@ variable "vnet_subnet_id" {
 }
 
 variable "service_principal_id" {
-  type = string
+  type    = string
   default = ""
 }
 
 variable "service_principal_secret" {
-  type = string
+  type    = string
   default = ""
 }
 
 variable "msi_enabled" {
-  type = bool
+  type    = bool
   default = false
 }
 

cluster/environments/azure-multiple-clusters-waf-tm-apimgmt/aks-variables.tf

@@ -13,7 +13,7 @@ variable "dns_prefix" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "ssh_public_key" {

cluster/environments/azure-multiple-clusters/aks-variables.tf

@@ -17,7 +17,7 @@ variable "dns_prefix" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "ssh_public_key" {

cluster/environments/azure-simple/variables.tf

@@ -52,7 +52,7 @@ variable "gitops_url_branch" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "resource_group_name" {

cluster/environments/azure-single-keyvault-cosmos-mongo-db-simple/variables.tf

@@ -72,7 +72,7 @@ variable "keyvault_resource_group" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "resource_group_name" {

cluster/environments/azure-single-keyvault/variables.tf

@@ -68,7 +68,7 @@ variable "keyvault_resource_group" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "resource_group_name" {

cluster/environments/azure-velero-restore/variables.tf

@@ -78,7 +78,7 @@ variable "keyvault_resource_group" {
 
 variable "kubernetes_version" {
   type    = string
-  default = "1.15.11"
+  default = "1.17.9"
 }
 
 variable "resource_group_name" {

docs/managed-identity.md

@@ -20,7 +20,7 @@ Grants AKS MI the ability to pull application images from ACR
 resource "azurerm_role_assignment" "acrpull" {
   scope                = var.acr_id
   role_definition_name = "AcrPull"
-  principal_id         = module.aks.kubelet_id
+  principal_id         = module.aks.kubelet_identity.client_id
 }
 ```
 
@@ -42,7 +42,7 @@ Grant AKS MI the ability to read and assign User Assigned MI
 resource "azurerm_role_assignment" "mi_operator" {
   scope                = data.azurerm_resource_group.kube_rg.id
   role_definition_name = "Managed Identity Operator"
-  principal_id         = module.aks.kubelet_id
+  principal_id         = module.aks.kubelet_identity.client_id
 }
 ```
 
@@ -54,7 +54,7 @@ Grant AKS MI the ability to manage VMs in AKS VM Scale Set
 resource "azurerm_role_assignment" "vm_contrib" {
   scope                = data.azurerm_resource_group.kube_rg.id
   role_definition_name = "Virtual Machine Contributor"
-  principal_id         = module.aks.kubelet_id
+  principal_id         = module.aks.kubelet_identity.client_id
 }
 ```
 

docs/single-cluster.md

@@ -202,7 +202,7 @@ variables:
   gitops_path: <insert value>
   keyvault_name: <insert value>
   keyvault_resource_group: <insert value>
-  kubernetes_version: 1.15.11
+  kubernetes_version: 1.17.9
   resource_group_name: <insert value>
   ssh_public_key: <insert value>
   service_principal_id: <insert value>
@@ -264,7 +264,7 @@ variables:
   vnet_name: 'myvnet'
   service_principal_id: '46b1b7dc-168a-ccc-bbb-aaaaaaa'
   service_principal_secret: 'aaaa-bbbb-43eb-9ead-dddddd'
-  kubernetes_version: '1.15.11'
+  kubernetes_version: '1.17.9'
   subnet_name: 'mysubnet'
   subnet_prefix: 10.8.0.0/24
   network_plugin: azure

test/bedrock_Azure_common_kv_test.go

@@ -24,7 +24,7 @@ func TestIT_Bedrock_AzureCommon_KV_Test(t *testing.T) {
 	addressSpace := "10.39.0.0/16"
 	kvName := k8sName + "-kv"
 	kvRG := kvName + "-rg"
-	k8sVersion := "1.15.11"
+	k8sVersion := "1.17.9"
 
 	location := os.Getenv("DATACENTER_LOCATION")
 	clientid := os.Getenv("ARM_CLIENT_ID")
@@ -44,34 +44,34 @@ func TestIT_Bedrock_AzureCommon_KV_Test(t *testing.T) {
 	copy.Copy("../cluster/environments/azure-common-infra", azureCommonInfraFolder)
 
 	// Remove any existing state
-        tfDir := azureCommonInfraFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
-        stateFiles, err := filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir := azureCommonInfraFolder + "/output"
-        if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob := azureCommonInfraFolder + "/*-flux"
-        fluxDirs, err := filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+	tfDir := azureCommonInfraFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
+	stateFiles, err := filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range stateFiles {
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir := azureCommonInfraFolder + "/output"
+	if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob := azureCommonInfraFolder + "/*-flux"
+	fluxDirs, err := filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, d := range fluxDirs {
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Create the resource group
 	cmd0 := exec.Command("az", "login", "--service-principal", "-u", clientid, "-p", clientsecret, "--tenant", tenantid)
@@ -129,34 +129,34 @@ func TestIT_Bedrock_AzureCommon_KV_Test(t *testing.T) {
 	copy.Copy("../cluster/environments/azure-single-keyvault", azureSingleKeyvaultFolder)
 
 	// Remove any existing state
-        tfDir = azureSingleKeyvaultFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob = azureSingleKeyvaultFolder + "/*tfstate*"
-        stateFiles, err = filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
+	tfDir = azureSingleKeyvaultFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob = azureSingleKeyvaultFolder + "/*tfstate*"
+	stateFiles, err = filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
 	for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir = azureSingleKeyvaultFolder + "/output"
-        if _, err = os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob = azureSingleKeyvaultFolder + "/*-flux"
-        fluxDirs, err = filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir = azureSingleKeyvaultFolder + "/output"
+	if _, err = os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob = azureSingleKeyvaultFolder + "/*-flux"
+	fluxDirs, err = filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
 	for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Create the aks resource group
 	cmd2 := exec.Command("az", "group", "create", "-n", k8sRG, "-l", location)

test/bedrock_Azure_mc_test.go

@@ -41,7 +41,7 @@ func TestIT_Bedrock_AzureMC_Test(t *testing.T) {
 	// Generate a common infra resources for integration use with azure multicluster environment
 	uniqueID := strings.ToLower(random.UniqueId())
 	k8sName := fmt.Sprintf("gtestk8s-%s", uniqueID)
-	k8sVersion := "1.15.11"
+	k8sVersion := "1.17.9"
 
 	location := os.Getenv("DATACENTER_LOCATION")
 	clientid := os.Getenv("ARM_CLIENT_ID")
@@ -64,35 +64,35 @@ func TestIT_Bedrock_AzureMC_Test(t *testing.T) {
 	azureCommonInfraFolder := "../cluster/test-temp-envs/azure-common-infra-" + k8sName
 	copy.Copy("../cluster/environments/azure-common-infra", azureCommonInfraFolder)
 
-        // Remove any existing state
-        tfDir := azureCommonInfraFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
-        stateFiles, err := filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir := azureCommonInfraFolder + "/output"
-        if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob := azureCommonInfraFolder + "/*-flux"
-        fluxDirs, err := filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+	// Remove any existing state
+	tfDir := azureCommonInfraFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
+	stateFiles, err := filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range stateFiles {
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir := azureCommonInfraFolder + "/output"
+	if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob := azureCommonInfraFolder + "/*-flux"
+	fluxDirs, err := filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, d := range fluxDirs {
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Create the common resource group
 	cmd0 := exec.Command("az", "login", "--service-principal", "-u", clientid, "-p", clientsecret, "--tenant", tenantid)
@@ -197,35 +197,35 @@ func TestIT_Bedrock_AzureMC_Test(t *testing.T) {
 	azureMultipleClustersFolder := "../cluster/test-temp-envs/azure-multiple-clusters-" + k8sName
 	copy.Copy("../cluster/environments/azure-multiple-clusters", azureMultipleClustersFolder)
 
-        // Remove any existing state
-        tfDir = azureMultipleClustersFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob = azureMultipleClustersFolder + "/*tfstate*"
-        stateFiles, err = filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir = azureMultipleClustersFolder + "/output"
-        if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob = azureMultipleClustersFolder + "/*-flux"
-        fluxDirs, err = filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+	// Remove any existing state
+	tfDir = azureMultipleClustersFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob = azureMultipleClustersFolder + "/*tfstate*"
+	stateFiles, err = filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range stateFiles {
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir = azureMultipleClustersFolder + "/output"
+	if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob = azureMultipleClustersFolder + "/*-flux"
+	fluxDirs, err = filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, d := range fluxDirs {
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Specify the test case folder and "-var" options
 	tfOptions := &terraform.Options{

test/bedrock_Azure_simple_test.go

@@ -29,7 +29,7 @@ func TestIT_Bedrock_AzureSimple_Test(t *testing.T) {
 	tenantId := os.Getenv("ARM_TENANT_ID")
 	dnsprefix := k8sName + "-dns"
 	k8sRG := k8sName + "-rg"
-	k8sVersion := "1.15.11"
+	k8sVersion := "1.17.9"
 	location := os.Getenv("DATACENTER_LOCATION")
 	publickey := os.Getenv("public_key")
 	sshkey := os.Getenv("ssh_key")

test/bedrock_Azure_single_cosmos_mongo_test.go

@@ -25,7 +25,7 @@ func TestIT_Bedrock_Azure_Single_KV_Cosmos_Mongo_DB_Test(t *testing.T) {
 	addressSpace := "10.39.0.0/16"
 	kvName := k8sName + "-kv"
 	kvRG := kvName + "-rg"
-	k8sVersion := "1.15.11"
+	k8sVersion := "1.17.9"
 	location := os.Getenv("DATACENTER_LOCATION")
 	tenantid := os.Getenv("ARM_TENANT_ID")
 	clientid := os.Getenv("ARM_CLIENT_ID")
@@ -43,35 +43,35 @@ func TestIT_Bedrock_Azure_Single_KV_Cosmos_Mongo_DB_Test(t *testing.T) {
 	azureCommonInfraFolder := "../cluster/test-temp-envs/azure-common-infra-" + k8sName
 	copy.Copy("../cluster/environments/azure-common-infra", azureCommonInfraFolder)
 
-        // Remove any existing state
-        tfDir := azureCommonInfraFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
-        stateFiles, err := filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir := azureCommonInfraFolder + "/output"
-        if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob := azureCommonInfraFolder + "/*-flux"
-        fluxDirs, err := filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+	// Remove any existing state
+	tfDir := azureCommonInfraFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob := azureCommonInfraFolder + "/*tfstate*"
+	stateFiles, err := filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range stateFiles {
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir := azureCommonInfraFolder + "/output"
+	if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob := azureCommonInfraFolder + "/*-flux"
+	fluxDirs, err := filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, d := range fluxDirs {
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Create the resource group
 	cmd0 := exec.Command("az", "login", "--service-principal", "-u", clientid, "-p", clientsecret, "--tenant", tenantid)
@@ -130,35 +130,35 @@ func TestIT_Bedrock_Azure_Single_KV_Cosmos_Mongo_DB_Test(t *testing.T) {
 	azureSingleKeyvaultFolder := "../cluster/test-temp-envs/azure-single-keyvault-cosmos-mongo-db-simple-" + k8sName
 	copy.Copy("../cluster/environments/azure-single-keyvault-cosmos-mongo-db-simple", azureSingleKeyvaultFolder)
 
-        // Remove any existing state
-        tfDir = azureSingleKeyvaultFolder + "/.terraform"
-        if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
-                os.RemoveAll(tfDir)
-        }
-        stateFileGlob = azureSingleKeyvaultFolder + "/*tfstate*"
-        stateFiles, err = filepath.Glob(stateFileGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, f := range stateFiles {
-                if err := os.Remove(f); err != nil {
-                        panic(err)
-                }
-        }
-        outputDir = azureSingleKeyvaultFolder + "/output"
-        if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
-                os.RemoveAll(outputDir)
-        }
-        fluxDirGlob = azureSingleKeyvaultFolder + "/*-flux"
-        fluxDirs, err = filepath.Glob(fluxDirGlob)
-        if err != nil {
-                panic(err)
-        }
-        for _, d := range fluxDirs {
-                if err := os.RemoveAll(d); err != nil {
-                        panic(err)
-                }
-        }
+	// Remove any existing state
+	tfDir = azureSingleKeyvaultFolder + "/.terraform"
+	if _, err := os.Stat(tfDir); !os.IsNotExist(err) {
+		os.RemoveAll(tfDir)
+	}
+	stateFileGlob = azureSingleKeyvaultFolder + "/*tfstate*"
+	stateFiles, err = filepath.Glob(stateFileGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range stateFiles {
+		if err := os.Remove(f); err != nil {
+			panic(err)
+		}
+	}
+	outputDir = azureSingleKeyvaultFolder + "/output"
+	if _, err := os.Stat(outputDir); !os.IsNotExist(err) {
+		os.RemoveAll(outputDir)
+	}
+	fluxDirGlob = azureSingleKeyvaultFolder + "/*-flux"
+	fluxDirs, err = filepath.Glob(fluxDirGlob)
+	if err != nil {
+		panic(err)
+	}
+	for _, d := range fluxDirs {
+		if err := os.RemoveAll(d); err != nil {
+			panic(err)
+		}
+	}
 
 	//Create the cluster resource group
 	cmd2 := exec.Command("az", "group", "create", "-n", k8sRG, "-l", location)