[stress testing] Move prod cluster to TME subscription (#9277)

* [stress testing] Move prod cluster to TME subscription

* [stress testing] Set up storage file data contributor role for cluster test identities

* Use fed auth service connection for stress pipeline deployment

* Updates to enable TME deployment

* Version/location updates for stress script usage

* Examples chart lock updates for stress

* Fix incorrect script path

eng/common/scripts/stress-testing/stress-test-deployment-lib.ps1

@@ -121,7 +121,7 @@ function DeployStressTests(
             Write-Warning "Overriding cluster group and subscription with defaults for 'prod' environment."
         }
         $clusterGroup = 'rg-stress-cluster-prod'
-        $subscription = 'Azure SDK Test Resources'
+        $subscription = 'Azure SDK Test Resources - TME'
     } elseif ($environment -eq 'storage') {
         if ($clusterGroup -or $subscription) {
             Write-Warning "Overriding cluster group and subscription with defaults for 'storage' environment."

eng/pipelines/templates/jobs/stress-cluster-provision.yml

@@ -17,12 +17,17 @@ jobs:
       name: azsdk-pool-mms-ubuntu-2204-general
       vmImage: ubuntu-22.04
     steps:
-      - pwsh: |
-          $subscriptionConfiguration = @'
-            $(sub-config-azure-cloud-test-resources)
-          '@ | ConvertFrom-Json -AsHashtable;
-
-          tools/stress-cluster/cluster/provision.ps1 `
+      - task: AzureCLI@2
+        displayName: Build and Deploy Stress Tests
+        inputs:
+          ${{ if eq(parameters.Environment, 'prod') }}:
+            azureSubscription: azure-sdk-tests-public
+          ${{ if eq(parameters.Environment, 'pg') }}:
+            azureSubscription: Azure SDK Playground
+          ${{ if eq(parameters.Environment, 'storage') }}:
+            azureSubscription: storage-sdk-stress-tests
+          scriptType: pscore
+          scriptPath: $(System.DefaultWorkingDirectory)/tools/stress-cluster/cluster/provision.ps1
+          arguments:
             -WhatIf:$${{ parameters.WhatIf }} `
-            -Environment ${{ parameters.Environment }} `
-            @subscriptionConfiguration
+            -Environment ${{ parameters.Environment }}

eng/pipelines/templates/jobs/stress-test-release.yml

@@ -68,7 +68,7 @@ jobs:
       displayName: Build and Deploy Stress Tests
       inputs:
         ${{ if eq(parameters.Environment, 'prod') }}:
-          azureSubscription: Azure SDK Test Resources
+          azureSubscription: azure-sdk-tests-public
         ${{ if eq(parameters.Environment, 'pg') }}:
           azureSubscription: Azure SDK Playground
         ${{ if eq(parameters.Environment, 'storage') }}:

tools/stress-cluster/chaos/examples/network-stress-example/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.2
-digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
-generated: "2024-05-23T11:38:32.810490735-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2024-11-01T18:05:45.572311938-04:00"

tools/stress-cluster/chaos/examples/network-stress-scenarios-example/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.2
-digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
-generated: "2024-05-23T11:38:01.807752664-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2024-11-01T18:07:35.664870135-04:00"

tools/stress-cluster/chaos/examples/parallel-pod-example/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.2
-digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
-generated: "2024-05-23T11:38:47.628996062-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2024-11-01T18:09:07.145743507-04:00"

tools/stress-cluster/chaos/examples/stress-debug-share-example/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.2
-digest: sha256:6eee71a7e8a4c0dc06d5fbbce39ef63237a0db0b7fc2da66e98e96b68985b764
-generated: "2024-05-23T11:38:19.251210631-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2024-11-01T18:07:17.501073902-04:00"

tools/stress-cluster/chaos/examples/stress-deployment-example/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: stress-test-addons
   repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.3
-digest: sha256:1cffb5ed8ea74953ab7611f9e2de2163af2c3f0918afb9928f71210da9c19a4a
-generated: "2024-10-02T16:18:41.429777815-04:00"
+  version: 0.3.4
+digest: sha256:e1d29df9556aaf06ee08fe1116695e6ce5396826b334b5c9d0f646da6599057f
+generated: "2024-11-01T18:07:24.379723401-04:00"

tools/stress-cluster/cluster/azure/cluster/acr.bicep

@@ -1,6 +1,8 @@
 param registryName string
 param location string = resourceGroup().location
 param objectIds array
+// Cluster may be in a tenant that does not include the ACR access groups
+param skipAcrRoleAssignment bool
 
 resource registry 'Microsoft.ContainerRegistry/registries@2019-12-01-preview' = {
   name: registryName
@@ -18,7 +20,7 @@ resource registry 'Microsoft.ContainerRegistry/registries@2019-12-01-preview' =
 }
 
 // Add AcrPush and AcrPull roles to access groups
-resource acrPushRole 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for objectId in objectIds: {
+resource acrPushRole 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for objectId in objectIds: if (!skipAcrRoleAssignment) {
   name: guid('azureContainerRegistryPushRole', objectId, resourceGroup().id)
   scope: registry
   properties: {
@@ -27,7 +29,7 @@ resource acrPushRole 'Microsoft.Authorization/roleAssignments@2020-04-01-preview
   }
 }]
 
-resource acrPullRole 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for objectId in objectIds: {
+resource acrPullRole 'Microsoft.Authorization/roleAssignments@2020-04-01-preview' = [for objectId in objectIds: if (!skipAcrRoleAssignment) {
   name: guid('azureContainerRegistryPullRole', objectId, resourceGroup().id)
   scope: registry
   properties: {

tools/stress-cluster/cluster/azure/cluster/cluster.bicep

@@ -6,6 +6,7 @@ param clusterName string
 param location string = resourceGroup().location
 param defaultAgentPoolMinNodes int = 6
 param defaultAgentPoolMaxNodes int = 20
+param defaultAgentPoolSku string = 'Standard_D8a_v4'
 param maintenanceWindowDay string = 'Monday'
 // AKS does not allow agentPool updates via existing managed cluster resources
 param updateNodes bool = false
@@ -38,7 +39,7 @@ var defaultAgentPool = {
   minCount: defaultAgentPoolMinNodes
   maxCount: defaultAgentPoolMaxNodes
   mode: 'User'
-  vmSize: 'Standard_D8a_v4'
+  vmSize: defaultAgentPoolSku
   type: 'VirtualMachineScaleSets'
   osType: 'Linux'
   osDiskType: 'Ephemeral'

tools/stress-cluster/cluster/azure/main.bicep

@@ -8,6 +8,8 @@ param clusterLocation string = 'westus3'
 param monitoringLocation string = 'centralus'
 param defaultAgentPoolMinNodes int = 6
 param defaultAgentPoolMaxNodes int = 20
+param defaultAgentPoolSku string = 'Standard_D8a_v4'
+param skipAcrRoleAssignment bool = false
 param maintenanceWindowDay string = 'Monday'
 param tags object
 // AKS does not allow agentPool updates via existing managed cluster resources
@@ -81,6 +83,7 @@ module cluster 'cluster/cluster.bicep' = {
         clusterName: clusterName
         defaultAgentPoolMinNodes: defaultAgentPoolMinNodes
         defaultAgentPoolMaxNodes: defaultAgentPoolMaxNodes
+        defaultAgentPoolSku: defaultAgentPoolSku 
         maintenanceWindowDay: maintenanceWindowDay 
         tags: tags
         groupSuffix: groupSuffix
@@ -95,6 +98,8 @@ module containerRegistry 'cluster/acr.bicep' = {
         registryName: '${replace(clusterName, '-', '')}${resourceSuffix}'
         location: clusterLocation
         objectIds: concat(accessGroups, array(cluster.outputs.kubeletIdentityObjectId))
+        // Cluster may be in a tenant that does not include the ACR access groups
+        skipAcrRoleAssignment: skipAcrRoleAssignment
     }
 }
 

tools/stress-cluster/cluster/azure/parameters/prod.json

@@ -3,7 +3,7 @@
   "contentVersion": "1.0.0.0",
   "parameters": {
     "subscriptionId": {
-      "value": "2cd617ea-1866-46b1-90e3-fffb087ebf9b"
+      "value": "4d042dc6-fe17-4698-a23f-ec6a8d1e98f4"
     },
     "groupSuffix": {
       "value": "prod"
@@ -23,6 +23,12 @@
     "defaultAgentPoolMaxNodes": {
       "value": 10
     },
+    "defaultAgentPoolSku": {
+      "value": "Standard_D8ds_v4"  // Standard_D8a_v4 does not have quota in TME tenant subscription
+    },
+    "skipAcrRoleAssignment": {
+      "value": true
+    },
     "maintenanceWindowDay": {
       "value": "Friday"
     },

tools/stress-cluster/cluster/kubernetes/stress-infrastructure/Chart.lock

@@ -3,7 +3,7 @@ dependencies:
   repository: https://charts.chaos-mesh.org
   version: 2.6.3
 - name: stress-test-addons
-  repository: https://stresstestcharts.blob.core.windows.net/helm/
-  version: 0.3.2
-digest: sha256:59235c0eac423267e28d9ac61392532ea74fb37a1be2567e4ac83277d62d8761
-generated: "2024-05-23T11:44:39.658622055-04:00"
+  repository: file:///home/ben/azs/azure-sdk-tools/tools/stress-cluster/cluster/kubernetes/stress-test-addons
+  version: 0.3.4
+digest: sha256:7ca87d5e4ce1b27a4bc0dbfa741a47739b0e343b7c2383c0e92b2dfaf4750a44
+generated: "2024-11-01T18:04:13.42983041-04:00"

tools/stress-cluster/cluster/kubernetes/stress-test-addons/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## 0.3.4 (2024-11-01)
+
+### Breaking Changes
+
+Changed prod cluster config values (cluster subscription/tenant migration)
+
 ## 0.3.3 (2024-07-10)
 
 ### Features Added

tools/stress-cluster/cluster/kubernetes/stress-test-addons/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: stress-test-addons
 description: Baseline resources and templates for stress testing clusters
 
-version: 0.3.3
+version: 0.3.4
 appVersion: v0.1

tools/stress-cluster/cluster/kubernetes/stress-test-addons/index.yaml

@@ -1,6 +1,15 @@
 apiVersion: v1
 entries:
   stress-test-addons:
+  - apiVersion: v2
+    appVersion: v0.1
+    created: "2024-11-01T18:04:40.435844341-04:00"
+    description: Baseline resources and templates for stress testing clusters
+    digest: b9c54a86c4d2fcd3ef38a4a24dea14ac18c8445b8e674b74d6c5fd4079e4914a
+    name: stress-test-addons
+    urls:
+    - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.3.4.tgz
+    version: 0.3.4
   - apiVersion: v2
     appVersion: v0.1
     created: "2024-07-10T15:50:11.882316755-04:00"
@@ -226,4 +235,4 @@ entries:
     urls:
     - https://stresstestcharts.blob.core.windows.net/helm/stress-test-addons-0.1.2.tgz
     version: 0.1.2
-generated: "2024-07-10T15:50:11.88175622-04:00"
+generated: "2024-11-01T18:04:40.430027237-04:00"

tools/stress-cluster/cluster/kubernetes/stress-test-addons/values.yaml

@@ -1,27 +1,27 @@
 env: test
 appInsightsKeySecretName:
   pg: appInsightsInstrumentationKey-s7b6dif73rup6
-  prod: appInsightsInstrumentationKey-xy7l7a2zaohws
+  prod: appInsightsInstrumentationKey-2c6jtbqddf52a
   dev: ""
   storage: appInsightsInstrumentationKey-4okf44ko4zuos
 appInsightsConnectionStringSecretName:
   pg: appInsightsConnectionString-s7b6dif73rup6
-  prod: appInsightsConnectionString-xy7l7a2zaohws
+  prod: appInsightsConnectionString-2c6jtbqddf52a
   dev: ""
   storage: appInsightsConnectionString-4okf44ko4zuos
 debugStorageKeySecretName:
   pg: debugStorageKey-s7b6dif73rup6
-  prod: debugStorageKey-xy7l7a2zaohws
+  prod: debugStorageKey-2c6jtbqddf52a
   dev: ""
   storage: debugStorageKey-4okf44ko4zuos
 debugStorageAccountSecretName:
   pg: debugStorageAccount-s7b6dif73rup6
-  prod: debugStorageAccount-xy7l7a2zaohws
+  prod: debugStorageAccount-2c6jtbqddf52a
   dev: ""
   storage: debugStorageAccount-4okf44ko4zuos
 debugFileShareName:
   pg: stressfiless7b6dif73rup6
-  prod: stressfilesxy7l7a2zaohws
+  prod: stressfiles2c6jtbqddf52a
   dev: ""
   storage: stressfiles4okf44ko4zuos
 staticTestSecretsKeyvaultName:
@@ -30,12 +30,12 @@ staticTestSecretsKeyvaultName:
   dev: ""
 clusterTestSecretsKeyvaultName:
   pg: stress-kv-s7b6dif73rup6
-  prod: stress-kv-xy7l7a2zaohws
+  prod: stress-kv-2c6jtbqddf52a
   dev: ""
   storage: stress-kv-4okf44ko4zuos
 secretProviderIdentity:
   pg: 0a7293d6-c5fa-47e7-a142-ef40bf6b6764
-  prod: c5454d90-811d-4d37-b685-f48d6f689aa3
+  prod: d9a57c7a-5f79-4aff-90e7-4ebdb24b81e6
   dev: ""
   storage: 96ff1075-09d3-406b-a1ae-81a470dd803a
 provisionerAppId:
@@ -49,12 +49,12 @@ infraWorkloadAppServiceAccountName:
   storage: workload-svc
 infraWorkloadAppClientId:
   pg: fb633f50-31c7-42af-9640-7651ab7cf69a
-  prod: daa992f7-ce2c-4ab5-8b51-b06e99caae1a
+  prod: 1b388c2c-1dc4-4456-a2a5-d5a4a5c7c4df
   dev: ""
   storage: 567c098d-734f-40d1-bacf-cba4c523f12c
 infraWorkloadAppObjectId:
   pg: 135cb549-37ce-4379-8738-39b981753256
-  prod: 9c705bcc-3878-4694-a34f-b1c0139315d6
+  prod: 0dea7f23-e8fe-4dbc-9980-400480ca9966
   dev: ""
   storage: b31f8eda-395f-4950-97b8-864925bfb8a5
 workloadAppClientNamePool:
@@ -64,7 +64,7 @@ workloadAppClientNamePool:
   storage: stress-app-workload-storage-0,stress-app-workload-storage-1,stress-app-workload-storage-2,stress-app-workload-storage-3,stress-app-workload-storage-4
 workloadAppIssuer:
   pg: https://westus3.oic.prod-aks.azure.com/72f988bf-86f1-41af-91ab-2d7cd011db47/c8b9b4a1-dee9-44e2-93d2-33fc5342ed26/
-  prod: https://westus2.oic.prod-aks.azure.com/72f988bf-86f1-41af-91ab-2d7cd011db47/5588aa6f-60c3-4504-8d1a-bbfda356780a/
+  prod: https://westus2.oic.prod-aks.azure.com/70a036f6-8e4d-4615-bad6-149c02e7720d/3a180dd0-9ec9-490e-a0b1-03fdbbca5198/
   dev: 
   storage: https://southcentralus.oic.prod-aks.azure.com/72f988bf-86f1-41af-91ab-2d7cd011db47/14f6afcc-b037-418e-b2c6-8d0726204adf/
 clusterGroup:
@@ -78,12 +78,12 @@ subscription:
   dev: public
 subscriptionId:
   pg: faa080af-c1d8-40ad-9cce-e1a450ca5b57
-  prod: 2cd617ea-1866-46b1-90e3-fffb087ebf9b
+  prod: 4d042dc6-fe17-4698-a23f-ec6a8d1e98f4
   dev: 
   storage: ba45b233-e2ef-4169-8808-49eb0d8eba0d
 tenantId:
   pg: 72f988bf-86f1-41af-91ab-2d7cd011db47
-  prod: 72f988bf-86f1-41af-91ab-2d7cd011db47
+  prod: 70a036f6-8e4d-4615-bad6-149c02e7720d
   dev: ""
   storage: 72f988bf-86f1-41af-91ab-2d7cd011db47
 

tools/stress-cluster/cluster/provision.ps1

@@ -6,17 +6,6 @@ param (
     # If provisioning an existing cluster and updating nodes, it must be done exclusively
     [switch]$UpdateNodes = $false,
 
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
-    [ValidateNotNullOrEmpty()]
-    [string] $TenantId,
-
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
-    [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')]
-    [string] $ProvisionerApplicationId,
-
-    [Parameter(ParameterSetName = 'Provisioner', Mandatory = $true)]
-    [string] $ProvisionerApplicationSecret,
-
     [ValidateScript({
         if (!(Test-Path $_)) {
             throw "LocalAddonsPath $LocalAddonsPath does not exist"
@@ -235,18 +224,12 @@ function main()
         throw "When using a custom environment you must set -LocalAddonsPath to provide the stress-infrastructure release with environment values"
     }
 
-    if ($PSCmdlet.ParameterSetName -eq 'Provisioner') {
-        az login `
-            --service-principal `
-            --username $ProvisionerApplicationId `
-            --password $ProvisionerApplicationSecret`
-            --tenant $TenantId
-        if ($LASTEXITCODE) { exit $LASTEXITCODE }
-    }
-
     if (!$Development) {
         $params = LoadEnvParams
         $STRESS_CLUSTER_RESOURCE_GROUP = "rg-stress-cluster-$($params.groupSuffix)"
+
+        RunOrExitOnFailure az account set -s $params.subscriptionId
+
         DeployClusterResources $params
         RegisterAKSFeatures $STRESS_CLUSTER_RESOURCE_GROUP $params.clusterName
     }