Merged PR 777594: Replace RunBxlWithPAT for reusable script in .azdo that uses task.setvariable instead of modifying the environment

The variables persist beyond the scope of the single task. This is useful to run RunBxlWithPAT in a pre-build task and then have the build task be exposed to these variables. This change looks like a net negative on the surface (why replace a single script with two separate pieces?) but we need it to migrate the pipelines to 1ESPT using the BuildXL workflow, which will call BuildXL outside of any wrapping script
2024-04-04 19:57:46 +00:00 · 2024-04-04 19:57:46 +00:00 · f76e0aded2
--- a/.azdo/common/set-variable-pats.yml
+++ b/.azdo/common/set-variable-pats.yml
@ -0,0 +1,19 @@
+# Sets pipeline variables with appropriate PATs to use as environment variables in internal selfhost builds 
+# Note this task assumes the variable group "BuildXL Secrets" is available
+parameters:
+- name: workingDirectory
+  type: string
+  default: $(Build.SourcesDirectory)
+
+steps:
+- task: PowerShell@2
+  displayName: Set environment for the job with necessary PATs 
+  inputs:
+    workingDirectory: ${{ parameters.workingDirectory }}
+    targetType: filePath
+    filePath: .azdo/scripts/SetVariablePats.ps1
+    arguments: >-
+      -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read)
+      -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read)
+      -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode)
+      -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
--- a/.azdo/distributed/selfhost-clean.yml
+++ b/.azdo/distributed/selfhost-clean.yml
@ -20,8 +20,6 @@ variables:
    value: $(Build.Repository.LocalPath)
  - name: BuildXL.LogsDirectory
    value: $(BuildXL.RepoDirectory)\Out\Logs
-  - name: PatArgs
-    value: -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
  - name: BxlScriptArgs
    value: -UseAdoBuildRunner -SharedCacheMode ConsumeAndPublish -Use Dev -DevRoot $(Domino.DogfoodPackagePath)\release\win-x64
  - name: BuildXLArgs
@ -82,20 +80,24 @@ jobs:
      netsh advfirewall firewall add rule name="Open Cache outbound port" protocol=TCP localport=7089 action=allow dir=OUT
    displayName: Setup firewall rules

-  - task: PowerShell@2
-    inputs:
-      targetType: filePath
-      filePath: 'RunBxlWithPAT.ps1'
-      arguments: '$(PatArgs) $(BxlScriptArgs) $(BuildXLArgs) /logsDirectory:$(BuildXL.LogsDirectory) /ado'
-      workingDirectory: $(BuildXL.RepoDirectory)
+  - template: /.azdo/common/set-variable-pats.yml
+
+  - script: bxl.cmd $(BxlScriptArgs) $(BuildXLArgs) /logsDirectory:$(BuildXL.LogsDirectory)
+    workingDirectory: $(BuildXL.RepoDirectory)
    displayName: 'Run clean selfhost build distributed'
    env:
      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
      AdoBuildRunnerWorkerPipelineRole: ${{ parameters.Role }}
      AdoBuildRunnerInvocationKey: SelfhostClean
      BlobCacheFactoryConnectionString: $(BuildXL-Selfhost-L3-ConnectionString)
+      1ESSHAREDASSETS_BUILDXL_FEED_PAT: $(1ESSHAREDASSETS_BUILDXL_FEED_PAT)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT)
+      MSENG_GIT_PAT: $(MSENG_GIT_PAT)
+      VSTSPERSONALACCESSTOKEN: $(VSTSPERSONALACCESSTOKEN)
+      ARTIFACT_CREDENTIALPROVIDERS_PATH: $(ARTIFACT_CREDENTIALPROVIDERS_PATH)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64)
+      VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: $(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)

-  
  - ${{ if eq(parameters.Role, 'Orchestrator') }}:
    - powershell: |
        $bxlWithRemotingLogDir = "$(BuildXL.LogsDirectory)"
--- a/.azdo/pr/macos-pr.yml
+++ b/.azdo/pr/macos-pr.yml
@ -13,12 +13,28 @@ jobs:

  - template: ../common/journaling.yml # Enable journaling
  
-  - task: PowerShell@2
-    displayName: Build MacOS Test Binaries
-    inputs:
-        targetType: filePath
-        filePath: ./RunBxlWithPat.ps1
-        arguments: /f:"output='$(Build.SourcesDirectory)\Out\Bin\Release\tests\*'" -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache) -CacheNamespace BuildXL.MacOS -SharedCacheMode ConsumeAndPublish /p:[Sdk.BuildXL]microsoftInternal=1 /q:ReleaseDotNetCoreMac /server- /logOutput:FullOutputOnWarningOrError /ado /scrubDirectory:Out\objects /cachemiss:[BxlMacOSPRValidation]
+  - template: /.azdo/common/set-variable-pats.yml
+  - script: >-
+      bxl.cmd
+      /f:output='$(Build.SourcesDirectory)\Out\Bin\Release\tests\*'
+      -CacheNamespace BuildXL.MacOS
+      -SharedCacheMode ConsumeAndPublish
+      /p:[Sdk.BuildXL]microsoftInternal=1
+      /q:ReleaseDotNetCoreMac
+      /server-
+      /logOutput:FullOutputOnWarningOrError
+      /ado
+      /scrubDirectory:Out\objects
+      /cachemiss:[BxlMacOSPRValidation]
+    displayName: 'Build MacOS Test Binaries'
+    env:
+      1ESSHAREDASSETS_BUILDXL_FEED_PAT: $(1ESSHAREDASSETS_BUILDXL_FEED_PAT)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT)
+      MSENG_GIT_PAT: $(MSENG_GIT_PAT)
+      VSTSPERSONALACCESSTOKEN: $(VSTSPERSONALACCESSTOKEN)
+      ARTIFACT_CREDENTIALPROVIDERS_PATH: $(ARTIFACT_CREDENTIALPROVIDERS_PATH)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64)
+      VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: $(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)

  - task: PublishBuildArtifacts@1
    displayName: Publish Test Binaries
--- a/.azdo/pr/public_pr.yml
+++ b/.azdo/pr/public_pr.yml
@ -140,22 +140,47 @@ jobs:
  displayName: Public PR Validation
  timeoutInMinutes: 120
  cancelTimeoutInMinutes: 1
+
  pool:
    name: BuildXL-DevOpsAgents-PME
  steps:
  - checkout: self
    fetchDepth: 1
+    fetchTags: false

  - template: ../common/journaling.yml # Enable journaling

  # Step 1. Build bits and deploy to dev
-  - task: PowerShell@2
-    displayName: BuildXL -Deploy Dev
-    inputs:
-      targetType: filePath
-      filePath: ./RunBxlWithPat.ps1
-      arguments: -minimal -deploy Dev -deployconfig Release -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache) -CacheNamespace BuildXL.Public -SharedCacheMode ConsumeAndPublish /p:[Sdk.BuildXL]microsoftInternal=1 /q:ReleaseNet8 /p:[BuildXL.Branding]SemanticVersion=$(BuildXL.SemanticVersion) /p:[BuildXL.Branding]PrereleaseTag=$(BuildXL.PreReleaseTag) /server- /logOutput:FullOutputOnWarningOrError /traceInfo:prvalidation=Public /ado /scrubDirectory:Out\objects /logsDirectory:$(BuildXL.LogsDirectory.LKG) /cachemiss:[BxlPublicValidation]
+  - template: /.azdo/common/set-variable-pats.yml

+  - script: >-
+      .\bxl.cmd
+      -minimal
+      -deploy Dev
+      -deployconfig Release
+      -CacheNamespace BuildXL.Public
+      -SharedCacheMode ConsumeAndPublish
+      /p:[Sdk.BuildXL]microsoftInternal=1
+      /q:ReleaseNet8
+      /p:[BuildXL.Branding]SemanticVersion=$(BuildXL.SemanticVersion)
+      /p:[BuildXL.Branding]PrereleaseTag=$(BuildXL.PreReleaseTag)
+      /server-
+      /logOutput:FullOutputOnWarningOrError
+      /traceInfo:prvalidation=Public
+      /ado
+      /scrubDirectory:Out\objects
+      /logsDirectory:$(BuildXL.LogsDirectory.LKG)
+      /cachemiss:[BxlPublicValidation]
+    displayName: BuildXL -Deploy Dev
+    env:
+      1ESSHAREDASSETS_BUILDXL_FEED_PAT: $(1ESSHAREDASSETS_BUILDXL_FEED_PAT)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT)
+      MSENG_GIT_PAT: $(MSENG_GIT_PAT)
+      VSTSPERSONALACCESSTOKEN: $(VSTSPERSONALACCESSTOKEN)
+      ARTIFACT_CREDENTIALPROVIDERS_PATH: $(ARTIFACT_CREDENTIALPROVIDERS_PATH)
+      CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64: $(CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64)
+      VSS_NUGET_EXTERNAL_FEED_ENDPOINTS: $(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)
+  
  - task: PublishPipelineArtifact@1
    displayName: Upload -Deploy Dev logs
    condition: always()
--- a/.azdo/scripts/SetVariablePats.ps1
+++ b/.azdo/scripts/SetVariablePats.ps1
@ -0,0 +1,51 @@
+<#
+.SYNOPSIS
+Sets secret variables with PATs for use in subsequent tasks in ADO
+#>
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [Parameter(mandatory=$true)]
+ [String]$OneEsPat,
+ [Parameter(mandatory=$true)]
+ [String]$CbPat,
+ [Parameter(mandatory=$false)]
+ [String]$NcPath,
+ [Parameter(mandatory=$true)]
+ [String]$MsEngGitPat,
+ [Parameter(mandatory=$false)]
+ [String]$VstsPat,
+ [Parameter(mandatory=$false)]
+ [String]$VstsCredProviderPath
+)
+
+# Sanity check: we should be running in an Azure Pipeline (let's avoid printing the values if not) 
+if (-not (Test-Path env:TF_BUILD)) {
+    throw "This script should only be run in an Azure Pipeline"
+}
+
+function Set-Variable($variableName, $variableValue) {
+    Write-Host "Setting $variableName"
+    [Environment]::SetEnvironmentVariable($variableName, $variableValue, "Process")     # This script is still used by RunBuildXLWithPAT so also make this available in the current environment
+    Write-Host "##vso[task.setvariable variable=$variableName;issecret=true]$variableValue"
+}
+
+Set-Variable "1ESSHAREDASSETS_BUILDXL_FEED_PAT" $OneEsPat
+Set-Variable "CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT" $CbPat
+Set-Variable "MSENG_GIT_PAT" $MsEngGitPat
+Set-Variable "VSTSPERSONALACCESSTOKEN" $VstsPat
+Set-Variable "ARTIFACT_CREDENTIALPROVIDERS_PATH" $VstsCredProviderPath
+
+# NPM authentication requires the PAT to be base64 encoded first
+$cbPatBytes = [System.Text.Encoding]::UTF8.GetBytes($CbPat)
+$b64CloudbuildPat = [Convert]::ToBase64String($cbPatBytes)
+
+# CODESYNC: Keep this variable name in sync with Public/Src/FrontEnd/UnitTests/Rush/IntegrationTests/RushIntegrationTestBase.cs
+Set-Variable "CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64" $b64CloudbuildPat
+
+if ($NcPath)
+{
+    Set-Variable "NUGET_CREDENTIALPROVIDERS_PATH" $NcPath
+}
+
+$vssEndpoints = "{`"endpointCredentials`": [{`"endpoint`":`"https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json`", `"password`":`"$OneEsPat`"}, {`"endpoint`":`"https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json`", `"password`":`"$CbPat`"}]}"
+Set-Variable "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS" $vssEndpoints
--- a/RunBxlWithPAT.ps1
+++ b/RunBxlWithPAT.ps1
@ -3,6 +3,9 @@

 Script for BuildXL self-hosting with specified PATs. This script is used to perform BuildXL self-hosting in Azure pipeline.

+NOTE: This script is deprecated. 
+      The way to run BuildXL on ADO is just including .azdo/common/set-variable-pats.yml to set up the environment
+      as a pre-build step of the 1ESPT BuildXL workflow (or before running bxl.cmd directly).
 #>

 [CmdletBinding(PositionalBinding=$false)]
@ -47,66 +50,33 @@ Param(
 [string[]]$BxlArgs
 )

-[Environment]::SetEnvironmentVariable("1ESSHAREDASSETS_BUILDXL_FEED_PAT", $OneEsPat, "Process")
-[Environment]::SetEnvironmentVariable("CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT", $CbPat, "Process")
-[Environment]::SetEnvironmentVariable("MSENG_GIT_PAT", $MsEngGitPat, "Process")
-[Environment]::SetEnvironmentVariable("VSTSPERSONALACCESSTOKEN", $VstsPat, "Process")
-[Environment]::SetEnvironmentVariable("ARTIFACT_CREDENTIALPROVIDERS_PATH", $VstsCredProviderPath, "Process")
+Write-Warning "This script is deprecated."
+Write-Warning "The way to run BuildXL on ADO is just including .azdo/common/set-variable-pats.yml to set up the environment as a pre-build step of the 1ESPT BuildXL workflow (or before running bxl.cmd directly)."

-# NPM authentication requires the PAT to be base64 encoded first
-$cbPatBytes = [System.Text.Encoding]::UTF8.GetBytes($CbPat)
-$b64CloudbuildPat = [Convert]::ToBase64String($cbPatBytes)
-# CODESYNC: Keep this variable name in sync with Public/Src/FrontEnd/UnitTests/Rush/IntegrationTests/RushIntegrationTestBase.cs
-[Environment]::SetEnvironmentVariable("CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT_B64", $b64CloudbuildPat, "Process")
-
-if ($NcPath)
-{
-    [Environment]::SetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH", $NcPath, "Process")
-}
-
-# CODESYNC: Keep this variable name in sync with Public/Src/FrontEnd/UnitTests/Nuget/Test.BuildXL.FrontEnd.Nuget.dsc
-[Environment]::SetEnvironmentVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "
-{
-    `"endpointCredentials`": [
-        {`"endpoint`":`"https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json`", `"password`":`"$OneEsPat`"}, 
-        {`"endpoint`":`"https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json`", `"password`":`"$CbPat`"}
-    ]
-}", "Process")
-
-$BxlCmdArgs = @(
-    "-Use", $Use,
-    "-DeployConfig", $DeployConfig,
-    "-DeployRuntime", $DeployRuntime
-    "-SharedCacheMode", $SharedCacheMode
+# 1. Set PATs
+$PatArgs = @(
+    "-OneEsPat", $OneEsPat,
+    "-CbPat", $CbPat,
+    "-MsEngGitPat", $MsEngGitPat
 )

-if (-not [string]::IsNullOrEmpty($Deploy))
+if (-not [string]::IsNullOrEmpty($NcPath))
 {
-    $BxlCmdArgs += @("-Deploy", $Deploy)
+    $PatArgs += @("-NcPath", $NcPath)
 }

-if ($Minimal)
+if (-not [string]::IsNullOrEmpty($VstsPat))
 {
-    $BxlCmdArgs += "-Minimal"
+    $PatArgs += @("-VstsPat", $VstsPat)
 }

-if ($EnableProcessRemoting)
+if (-not [string]::IsNullOrEmpty($VstsCredProviderPath))
 {
-    $BxlCmdArgs += "-EnableProcessRemoting"
+    $PatArgs += @("-VstsCredProviderPath", $VstsCredProviderPath)
 }

-if (-not [string]::IsNullOrEmpty($AnyBuildClientDir))
-{
-    $BxlCmdArgs += @("-AnyBuildClientDir", "$AnyBuildClientDir")
-}
+$PatArgsStr = $PatArgs -Join " "
+Invoke-Expression ".azdo/scripts/SetVariablePats.ps1 $PatArgsStr"

-if (-not [string]::IsNullOrEmpty($CacheNamespace))
-{
-    $BxlCmdArgs += @("-CacheNamespace", "$CacheNamespace")
-}
-
-$BxlCmdArgs += $BxlArgs
-
-Write-Host "Call bxl.cmd $BxlCmdArgs"
-
-.\bxl.cmd $BxlCmdArgs
+Write-Host "Call bxl.cmd $BxlArgs"
+.\bxl.cmd $BxlArgs