Use SAS tokens to get an identity in Connectivity tests (#7313)

This PR introduces the use of short-lived SAS tokens to get an identity using the AzureCLI task. The use of shared keys to access the blob storage account is now discouraged, prompting this change.

We don't always upload logs to the storage account, only when we want to debug something. However, log upload is currently broken after the shared key on the storage account was disabled.

This is the manually triggered test run for this PR:

![image](https://github.com/Azure/iotedge/assets/90283547/a1d6b031-cdbb-4a45-b002-e581108bffa8)



- [X] I have read the [contribution guidelines](https://github.com/azure/iotedge#contributing).

builds/e2e/connectivity.yaml

@@ -30,11 +30,16 @@ resources:
     branch: 'main'
 
 jobs:
+  - template: templates/get-storage-uri.yaml
+    parameters:
+      azureSubscription: $(azure.subscription)
+
 ################################################################################
   - job: linux_amd64_moby
 ################################################################################
     displayName: Linux AMD64 Moby
-    condition: and(eq(variables['run.linux.amd64.moby'], 'true'), ne(variables['agent.group'], ''))
+    dependsOn: Token
+    condition: and(succeeded('Token'), eq(variables['run.linux.amd64.moby'], 'true'), ne(variables['agent.group'], ''))
     timeoutInMinutes: 240
     strategy:
       matrix:
@@ -94,6 +99,7 @@ jobs:
       edgelet.artifact.name: 'iotedged-ubuntu22.04-amd64'
       aziotis.artifact.name: 'packages_ubuntu-22.04_amd64'
       aziotis.package.filter: 'aziot-identity-service_*_amd64.deb'
+      sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ]
     steps:
       - task: Bash@3
         name: Print_test_parameters
@@ -125,7 +131,6 @@ jobs:
             edgebuilds-azurecr-io-pwd,
             kvLogAnalyticWorkspaceId,
             kvLogAnalyticSharedKey,
-            EdgeConnectivityStorageAccountConnString,
             GitHubAccessToken
       - task: AzureKeyVault@1
         displayName: 'Azure Key Vault: $(azure.keyVault)'
@@ -208,7 +213,7 @@ jobs:
           logAnalyticsWorkspaceId: '$(kvLogAnalyticWorkspaceId)'
           logAnalyticsSharedKey: '$(kvLogAnalyticSharedKey)'
           testResultCoordinator.logAnalyticsLogType: '$(testResultCoordinator.logAnalyticsLogType)'
-          testResultCoordinator.storageAccountConnectionString: '$(EdgeConnectivityStorageAccountConnString)'
+          testResultCoordinator.blobStorageAccountUriWithSasToken: '$(sas_uri)'
           metricsCollector.metricsEndpointsCSV: '$(metricsCollector.metricsEndpointsCSV)'
           metricsCollector.scrapeFrequencyInSecs: '$(metricsCollector.scrapeFrequencyInSecs)'
           metricsCollector.uploadTarget: '$(metricsCollector.uploadTarget)'
@@ -225,7 +230,8 @@ jobs:
   - job: linux_arm32v7_moby
 ################################################################################
     displayName: Linux ARM32v7 Moby
-    condition: and(eq(variables['run.linux.arm32v7.moby'], 'true'), ne(variables['agent.group'], ''))
+    dependsOn: Token
+    condition: and(succeeded('Token'), eq(variables['run.linux.arm32v7.moby'], 'true'), ne(variables['agent.group'], ''))
     timeoutInMinutes: 240 
     strategy:
       matrix:
@@ -287,6 +293,7 @@ jobs:
         - run-connectivity -equals true
     variables:
       edgelet.artifact.name: 'iotedged-debian11-arm32v7'
+      sas_uri: $[ dependencies.Token.outputs['generate.sas_uri'] ]
     steps:
       - task: Bash@3
         name: Print_test_parameters
@@ -318,7 +325,6 @@ jobs:
             edgebuilds-azurecr-io-pwd,
             kvLogAnalyticWorkspaceId,
             kvLogAnalyticSharedKey,
-            EdgeConnectivityStorageAccountConnString,
             GitHubAccessToken
       - task: AzureKeyVault@1
         displayName: 'Azure Key Vault: $(azure.keyVault)'
@@ -387,7 +393,7 @@ jobs:
           logAnalyticsWorkspaceId: '$(kvLogAnalyticWorkspaceId)'
           logAnalyticsSharedKey: '$(kvLogAnalyticSharedKey)'
           testResultCoordinator.logAnalyticsLogType: '$(testResultCoordinator.logAnalyticsLogType)'
-          testResultCoordinator.storageAccountConnectionString: '$(EdgeConnectivityStorageAccountConnString)'
+          testResultCoordinator.blobStorageAccountUriWithSasToken: '$(sas_uri)'
           metricsCollector.metricsEndpointsCSV: '$(metricsCollector.metricsEndpointsCSV)'
           metricsCollector.scrapeFrequencyInSecs: '$(metricsCollector.scrapeFrequencyInSecs)'
           metricsCollector.uploadTarget: '$(metricsCollector.uploadTarget)'

builds/e2e/templates/connectivity-deploy.yaml

@@ -11,6 +11,7 @@ parameters:
   container.registry: ''
   container.registry.username: ''
   container.registry.password: ''
+  testResultCoordinator.blobStorageAccountUriWithSasToken: ''
   iotHub.connectionString: ''
   eventHub.connectionString: ''
   upstream.protocol: ''
@@ -20,7 +21,6 @@ parameters:
   logAnalyticsWorkspaceId: ''
   logAnalyticsSharedKey: ''
   testResultCoordinator.logAnalyticsLogType: ''
-  testResultCoordinator.storageAccountConnectionString: ''
   metricsCollector.metricsEndpointsCSV: ''
   metricsCollector.scrapeFrequencyInSecs: ''
   metricsCollector.uploadTarget: ''
@@ -129,10 +129,10 @@ steps:
           -metricsEndpointsCSV "${{ parameters['metricsCollector.metricsEndpointsCSV'] }}" \
           -metricsScrapeFrequencyInSecs "${{ parameters['metricsCollector.scrapeFrequencyInSecs'] }}" \
           -metricsUploadTarget "${{ parameters['metricsCollector.uploadTarget'] }}" \
+          -blobStorageAccountUriWithSasToken "${{ parameters['testResultCoordinator.blobStorageAccountUriWithSasToken'] }}" \
           -deploymentFileName "${{ parameters['deploymentFileName'] }}" \
           -EdgeHubRestartTestRestartPeriod "${{ parameters['edgeHubRestartTest.restartPeriod'] }}" \
           -EdgeHubRestartTestSdkOperationTimeout "${{ parameters['edgeHubRestartTest.sdkOperationTimeout'] }}" \
-          -storageAccountConnectionString "${{ parameters['testResultCoordinator.storageAccountConnectionString'] }}" \
           -devOpsAccessToken "$(System.AccessToken)" \
           -devOpsBuildId "$(Build.BuildId)" \
           -edgeRuntimeBuildNumber "${{ parameters['edgeRuntime.buildNumber'] }}" \

builds/e2e/templates/get-storage-uri.yaml

@@ -5,6 +5,10 @@ parameters:
 jobs:
   - job: Token
     displayName: 'Get SAS URI for Blob Storage Account'
+    pool:
+      name: $(pool.linux.name)
+      demands:
+        - ImageOverride -equals agent-aziotedge-ubuntu-22.04-msmoby
     steps:
       - task: AzureCLI@2
         name: generate

e2e_deployment_files/connectivity_deployment.template.json

@@ -388,8 +388,8 @@
               "logUploadEnabled": {
                 "value": "<TestResultCoordinator.logUploadEnabled>"
               },
-              "STORAGE_ACCOUNT_CONNECTION_STRING": {
-                "value": "<TestResultCoordinator.StorageAccountConnectionString>"
+              "BLOB_STORE_SAS": {
+                "value": "<testBlobStoreSas>"
               },
               "NetworkControllerRunProfile": {
                 "value": "<NetworkController.RunProfile>"

scripts/linux/trcE2ETest.sh

@@ -36,7 +36,7 @@ function usage() {
     echo ' -deploymentFileName                      Deployment file name'
     echo ' -EdgeHubRestartTestRestartPeriod         EdgeHub restart period (must be greater than 1 minutes)'
     echo ' -EdgeHubRestartTestSdkOperationTimeout   SDK retry timeout'
-    echo ' -storageAccountConnectionString          Azure storage account connection string with privilege to create blob container.'
+    echo ' -blobStorageAccountUriWithSasToken       Azure storage account blob store SAS Uri.'
     echo ' -edgeRuntimeBuildNumber                  Build number for specifying edge runtime (edgeHub and edgeAgent)'
     echo ' -testRuntimeLogLevel                     RuntimeLogLevel given to Quickstart, which is given to edgeAgent and edgeHub.'
     echo ' -testInfo                                Contains comma delimiter test information, e.g. build number and id, source branches of build, edgelet and images.'
@@ -193,7 +193,7 @@ function prepare_test_from_artifacts() {
     sed -i -e "s@<OptimizeForPerformance>@$optimize_for_performance@g" "$deployment_working_file"
     sed -i -e "s@<TestResultCoordinator.LogAnalyticsLogType>@$LOG_ANALYTICS_LOGTYPE@g" "$deployment_working_file"
     sed -i -e "s@<TestResultCoordinator.logUploadEnabled>@$log_upload_enabled@g" "$deployment_working_file"
-    sed -i -e "s@<TestResultCoordinator.StorageAccountConnectionString>@$STORAGE_ACCOUNT_CONNECTION_STRING@g" "$deployment_working_file"
+    sed -i -e "s@<testBlobStoreSas>@$BLOB_STORE_SAS@Ig" "$deployment_working_file"
     sed -i -e "s@<TestInfo>@$TEST_INFO@g" "$deployment_working_file"
 
     sed -i -e "s@<NetworkController.RunProfile>@$NETWORK_CONTROLLER_RUNPROFILE@g" "$deployment_working_file"
@@ -411,7 +411,7 @@ function process_args() {
             METRICS_UPLOAD_TARGET="$arg"
             saveNextArg=0
         elif [ $saveNextArg -eq 24 ]; then
-            STORAGE_ACCOUNT_CONNECTION_STRING="$arg"
+            BLOB_STORE_SAS="$arg"
             saveNextArg=0
         elif [ $saveNextArg -eq 25 ]; then
             DEVOPS_ACCESS_TOKEN="$arg"
@@ -514,7 +514,7 @@ function process_args() {
                 '-metricsEndpointsCSV' ) saveNextArg=21;;
                 '-metricsScrapeFrequencyInSecs' ) saveNextArg=22;;
                 '-metricsUploadTarget' ) saveNextArg=23;;
-                '-storageAccountConnectionString' ) saveNextArg=24;;
+                '-blobStorageAccountUriWithSasToken' ) saveNextArg=24;;
                 '-devOpsAccessToken' ) saveNextArg=25;;
                 '-devOpsBuildId' ) saveNextArg=26;;
                 '-deploymentFileName' ) saveNextArg=27;;
@@ -565,7 +565,7 @@ function process_args() {
     [[ -z "$METRICS_ENDPOINTS_CSV" ]] && { print_error 'Metrics endpoints csv is required'; exit 1; }
     [[ -z "$METRICS_SCRAPE_FREQUENCY_IN_SECS" ]] && { print_error 'Metrics scrape frequency is required'; exit 1; }
     [[ -z "$METRICS_UPLOAD_TARGET" ]] && { print_error 'Metrics upload target is required'; exit 1; }
-    [[ -z "$STORAGE_ACCOUNT_CONNECTION_STRING" ]] && { print_error 'Storage account connection string is required'; exit 1; }
+    [[ -z "$BLOB_STORE_SAS" ]] && { print_error 'Blob Store SAS Uri is required'; exit 1; }
     [[ -z "$TEST_INFO" ]] && { print_error 'Test info is required'; exit 1; }
     [[ -z "$REPO_PATH" ]] && { print_error 'Repo path is required'; exit 1; }
     [[ (-z "${TEST_NAME,,}") || ("${TEST_NAME,,}" != "${LONGHAUL_TEST_NAME,,}" && "${TEST_NAME,,}" != "${CONNECTIVITY_TEST_NAME,,}") ]] && { print_error 'Invalid test name'; exit 1; }

test/modules/TestResultCoordinator/Services/TestResultReportingService.cs

@@ -105,10 +105,7 @@ namespace TestResultCoordinator.Services
             {
                 try
                 {
-                    Uri blobContainerWriteUriForLog = await TestReportUtil.GetOrCreateBlobContainerSasUriForLogAsync(this.serviceSpecificSettings.StorageAccountConnectionString);
-                    blobContainerUri = $"{blobContainerWriteUriForLog.Scheme}{Uri.SchemeDelimiter}{blobContainerWriteUriForLog.Authority}{blobContainerWriteUriForLog.AbsolutePath}";
-
-                    await TestReportUtil.UploadLogsAsync(Settings.Current.IoTHubConnectionString, blobContainerWriteUriForLog, this.logUploadDuration, this.logger);
+                    await TestReportUtil.UploadLogsAsync(Settings.Current.IoTHubConnectionString, this.serviceSpecificSettings.BlobStorageAccountUriWithSasToken, this.logUploadDuration, this.logger);
                 }
                 catch (Exception ex)
                 {

test/modules/TestResultCoordinator/Settings.cs

@@ -46,7 +46,7 @@ namespace TestResultCoordinator
             TimeSpan verificationDelay,
             TimeSpan sendReportFrequency,
             bool logUploadEnabled,
-            string storageAccountConnectionString,
+            string blobStorageAccountUriWithSasToken,
             string networkControllerRunProfileName,
             ushort unmatchedResultsMaxSize,
             string testInfo,
@@ -71,7 +71,7 @@ namespace TestResultCoordinator
             {
                 this.TestResultReportingServiceSettings = Option.Some(new TestResultReportingServiceSettings()
                 {
-                    StorageAccountConnectionString = Preconditions.CheckNonWhiteSpace(storageAccountConnectionString, nameof(storageAccountConnectionString)),
+                    BlobStorageAccountUriWithSasToken = new Uri(Preconditions.CheckNonWhiteSpace(blobStorageAccountUriWithSasToken, nameof(blobStorageAccountUriWithSasToken))),
                     LogAnalyticsLogType = Preconditions.CheckNonWhiteSpace(logAnalyticsLogType, nameof(logAnalyticsLogType)),
                     LogAnalyticsSharedKey = Preconditions.CheckNonWhiteSpace(logAnalyticsSharedKey, nameof(logAnalyticsSharedKey)),
                     LogAnalyticsWorkspaceId = Preconditions.CheckNonWhiteSpace(logAnalyticsWorkspaceId, nameof(logAnalyticsWorkspaceId)),
@@ -164,7 +164,7 @@ namespace TestResultCoordinator
                 configuration.GetValue("verificationDelay", TimeSpan.FromMinutes(15)),
                 configuration.GetValue("sendReportFrequency", TimeSpan.FromHours(24)),
                 configuration.GetValue<bool>("logUploadEnabled", true),
-                configuration.GetValue<string>("STORAGE_ACCOUNT_CONNECTION_STRING"),
+                configuration.GetValue<string>("BLOB_STORE_SAS"),
                 configuration.GetValue<string>(TestConstants.NetworkController.RunProfilePropertyName),
                 configuration.GetValue<ushort>("UNMATCHED_RESULTS_MAX_SIZE", DefaultUnmatchedResultsMaxSize),
                 configuration.GetValue<string>("TEST_INFO"),
@@ -282,7 +282,7 @@ namespace TestResultCoordinator
 
     internal struct TestResultReportingServiceSettings
     {
-        public string StorageAccountConnectionString;
+        public Uri BlobStorageAccountUriWithSasToken;
         public string LogAnalyticsWorkspaceId;
         public string LogAnalyticsSharedKey;
         public string LogAnalyticsLogType;