Merged PR 647318: Pipeline exercising BuildXL remoting capability

Currently it only remotes process pip with "compile" tag. The remoting configuration is still not optimal currently, particularly for the beefy pool we have (32 cores). In this run, https://dev.azure.com/mseng/Domino/_build/results?buildId=16779538&view=results, with remoting multiplier 1.5, and num of leases is assumed to be 2*maxproc, there are 42 processes going to AnyBuild, but 22 of them failed to find agents because AnyBuild is over saturated. In this run, https://dev.azure.com/mseng/Domino/_build/results?buildId=16779745&view=results, with remoting multiplier 1.25 and num of leases 24, there are only 17 processes going to AnyBuild, and all of them successfully execute remotely. More work is still needed for optimal configuration, e.g., - Feedback from AnyBuild about available leases, etc. Related work items: #1912875
2022-02-02 21:27:11 +00:00 · 2022-02-02 21:27:11 +00:00 · 76adb87e41
--- a/.azdo/remoting/az-login.yml
+++ b/.azdo/remoting/az-login.yml
@ -0,0 +1,16 @@
+parameters:
+- name: SPAppId
+  type: string
+  default: $(BuildXLPipelinesAppId)
+- name: SPPassword
+  type: string
+  default: $(AzureApp-BuildXL-Pipelines)
+- name: SPTenant
+  type: string
+  default: $(BuildXLPipelinesTenantId)
+
+steps:
+- script: |
+    call az login --service-principal --username ${{ parameters.SPAppId }} --password ${{ parameters.SPPassword }} --tenant ${{ parameters.SPTenant }}
+    if ERRORLEVEL 1 echo "az login failed" & exit /b 1
+  displayName: 'Azure login'
--- a/.azdo/remoting/install-client.yml
+++ b/.azdo/remoting/install-client.yml
@ -0,0 +1,37 @@
+parameters:
+- name: AnyBuildSource
+  type: string
+  default: ''
+- name: Ring
+  type: string
+  default: 'Dogfood'
+
+steps:
+- powershell: |
+   Remove-Item -Force -Recurse "$env:LOCALAPPDATA\Microsoft\AnyBuild" -ea SilentlyContinue
+   
+   # Make lowercase as AzStorage cannot handle uppercase in URL.
+   $source = "${{ parameters.AnyBuildSource }}".ToLowerInvariant()
+   
+   $bootstrapperArgs = @("$source", "${{ parameters.Ring }}")
+
+   Write-Host "Bootstrapper args: '$bootstrapperArgs'";
+   
+   while ($true)
+   {
+     $script = ((curl.exe -s -S --retry 10 --retry-connrefused "$source/bootstrapper.ps1") | Out-String)
+     Write-Host "Downloaded script: `r`n $script";
+     if ($LASTEXITCODE -eq 0)
+     {
+         break;
+     }
+   
+     # We sometimes get a 404 error when downloading the bootstrapper
+     # while the bootstrapper script is updating. Needs an eventual fix instead of this workaround.
+     Write-Host "ERROR: Failed downloading the bootstrapper script. Trying again."
+   }
+   
+   Invoke-Command -ScriptBlock ([scriptblock]::Create($script)) -ArgumentList $bootstrapperArgs
+  failOnStderr: true
+  displayName: 'Install AnyBuild client'
+  continueOnError: false
--- a/.azdo/remoting/pipeline.yml
+++ b/.azdo/remoting/pipeline.yml
@ -0,0 +1,85 @@
+trigger: none
+
+pool:
+  name: BuildXL-DevOpsAgents-PME
+
+variables:
+  NugetCredentialProviderPath: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\bin\Debug\netcoreapp2.1\win-x64'
+  Datacenter: 'westus2'
+  ClusterName: 'Bxl1'
+  RemoteUri: 'https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191'
+  ClientContainerUrl: 'https://anybuild$(ClusterName)$(Datacenter).blob.core.windows.net/clientreleases'
+  BootstrapArgs: '-DeployDev /p:[Sdk.BuildXL]microsoftInternal=1'
+  RemotingArgs: '-UseDev -Minimal -EnableProcessRemoting -RemoteServiceUri $(RemoteUri) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID=$(BuildXLPipelinesAppId) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV=AnyBuildServicePrincipalPwd /numRemoteAgentLeases:24 /remotingThresholdMultiplier:1.25 /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=*'
+
+steps:
+
+- checkout: self
+  fetchDepth: 1
+
+- template: ./az-login.yml
+
+- script: 'C:\windows\system32\fsutil usn createjournal m=0x20000000 a=0x8000000 D:'
+  displayName: 'Enable Journaling for D drive'
+
+- script: |
+   if EXIST %AppData%\NuGet\NuGet.Config (
+       del %AppData%\NuGet\NuGet.Config
+   )
+  displayName: 'Nuget Hack: delete old nuget.config first'
+
+- task: UseDotNet@2
+  displayName: 'Use .NET Core sdk 6.x'
+  inputs:
+    version: 6.x
+
+- task: DotNetCoreCLI@2
+  displayName: 'Build BuildXL.Tools.CredentialProvider'
+  inputs:
+    projects: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\BuildXL.Tools.CredentialProvider.csproj'
+    arguments: '-r win-x64'
+
+- template: ./install-client.yml
+  parameters:
+    AnyBuildSource: $(ClientContainerUrl)
+
+- task: BatchScript@1
+  displayName: 'Kill existing BuildXL processes'
+  inputs:
+    filename: Shared/Scripts/KillBxlInstancesInRepo.cmd
+  continueOnError: true
+  condition: always()
+
+- task: PowerShell@2
+  displayName: 'Build BXL'
+  inputs:
+    targetType: filePath
+    filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
+    arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(BootstrapArgs)'
+
+- task: PowerShell@2
+  displayName: 'Run BXL with remoting'
+  inputs:
+    targetType: filePath
+    filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
+    arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(RemotingArgs)'
+  env:
+    AnyBuildServicePrincipalPwd: $(AzureApp-BuildXL-Pipelines)
+
+- task: BatchScript@1
+  displayName: 'Kill existing BuildXL processes'
+  inputs:
+    filename: Shared/Scripts/KillBxlInstancesInRepo.cmd
+  continueOnError: true
+  condition: always()
+
+- task: PublishPipelineArtifact@1
+  displayName: 'Upload logs'
+  inputs:
+    targetPath: '$(Build.SourcesDirectory)\Out\Logs'
+  condition: always()
+
+- template: ./validate.yml
+  parameters:
+    LogDirectory: '$(Build.SourcesDirectory)\Out\Logs'
+
--- a/.azdo/remoting/validate.yml
+++ b/.azdo/remoting/validate.yml
@ -0,0 +1,32 @@
+parameters:
+- name: LogDirectory
+  type: string
+  default: '$(Build.SourcesDirectory)\Out\Logs'
+
+steps:
+- powershell: |
+    $f = Get-ChildItem -Path '${{ parameters.LogDirectory }}\*\AnyBuildDaemonManager.log' -Recurse  | Select-Object -First 1
+    $bxlWithRemotingLogDir = $f.DirectoryName
+    $statsContent = Get-Content -Path (Join-Path $bxlWithRemotingLogDir "BuildXL.stats") | Out-String
+    $stats = ConvertFrom-StringData -StringData $statsContent
+    $runRemoteCount = $stats.Get_Item("PipExecution.TotalRunRemoteProcesses")
+    $runLocalFallbackCount = $stats.Get_Item("PipExecution.TotalRemoteFallbackRetries")
+
+    Write-Host "Run remote: $runRemoteCount"
+    Write-Host "Run local due to fallback: $runLocalFallbackCount"
+
+    if ($runRemoteCount -eq 0)
+    {
+        Write-Host "No process pip ran remotely"
+        exit 1
+    }
+
+    if ($runLocalFallbackCount -ne 0)
+    {
+        Write-Host "$runLocalFallbackCount remoted process pip(s) ran locally"
+        exit 1
+    }
+
+    exit 0
+  continueOnError: true
+  displayName: 'Validate remoting'
--- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs
+++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs
@ -118,7 +118,8 @@ namespace BuildXL.Processes.Remoting
                        closeDaemonOnDispose: true,
                        m_executionContext.CancellationToken,
                        logDirectory: logDir,
-                        additionalAnyBuildParameters: extraParams);
+                        additionalAnyBuildParameters: extraParams,
+                        inheritHandlesOnProcessCreation: false);
                }
                catch (Exception e)
                {
@ -165,6 +166,20 @@ namespace BuildXL.Processes.Remoting
                $"--CacheDir {localCacheDir}",
            };

+            if (!string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId))
+            {
+                args.Add("--ClientApplicationId");
+                args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId);
+
+                if (string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv))
+                {
+                    throw new BuildXLException($"Service principal password is required for starting AnyBuild daemon");
+                }
+
+                args.Add("--ClientSecretEnvironmentVariable");
+                args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv);
+            }
+
            if (!string.IsNullOrEmpty(m_configuration.Schedule.RemoteExecutionServiceUri))
            {
                try
--- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs
+++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs
@ -41,9 +41,8 @@ namespace BuildXL.Processes.Remoting.AnyBuild
                ExitCode = result.ExitCode,
                StdOut = result.StdOut,
                StdErr = result.StdErr,
-                Disposition = result.Disposition switch
+                Disposition = result.ExecutionDisposition switch
                {
-                    CommandExecutionDisposition.Unknown => RemoteResultDisposition.Unknown,
                    CommandExecutionDisposition.CacheHit => RemoteResultDisposition.CacheHit,
                    CommandExecutionDisposition.Remoted => RemoteResultDisposition.Remoted,
                    CommandExecutionDisposition.RanLocally => RemoteResultDisposition.RanLocally,
--- a/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs
+++ b/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs
@ -8,11 +8,6 @@ namespace BuildXL.Processes.Remoting
    /// </summary>
    public enum RemoteResultDisposition
    {
-        /// <summary>
-        /// Default zero value.
-        /// </summary>
-        Unknown,
-
        /// <summary>
        /// Process completed with cache hit.
        /// </summary>
--- a/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
+++ b/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
@ -88,6 +88,16 @@ namespace BuildXL.Utilities.Configuration
        /// </summary>
        public static readonly Setting<string> AnyBuildExtraArgs = CreateSetting("BUILDXL_ANYBUILD_EXTRA_ARGS", value => value);

+        /// <summary>
+        /// Service principal app id for starting AnyBuild daemon.
+        /// </summary>
+        public static readonly Setting<string> AnyBuildServicePrincipalAppId = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID", value => value);
+
+        /// <summary>
+        /// Environment variable where the service principal password for starting AnyBuild daemon is stored.
+        /// </summary>
+        public static readonly Setting<string> AnyBuildServicePrincipalPwdEnv = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV", value => value);
+
        /// <summary>
        /// Indicates whether the application should fail fast on null reference exceptions
        /// </summary>
--- a/RunBxlWithPAT.ps1
+++ b/RunBxlWithPAT.ps1
@ -0,0 +1,82 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [Parameter(mandatory=$true)]
+ [String]$OneEsPat,
+ [Parameter(mandatory=$true)]
+ [String]$CbPat,
+ [Parameter(mandatory=$true)]
+ [String]$NcPath,
+ [Parameter(mandatory=$true)]
+ [String]$MsEngGitPat,
+
+ [Parameter(Mandatory=$false)]
+ [switch]$DeployDev = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$UseDev = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$Minimal = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$Release = $false,
+
+ [Parameter(Mandatory=$false)]
+ [switch]$EnableProcessRemoting = $false,
+ [Parameter(Mandatory=$false)]
+ [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
+ [Parameter(Mandatory=$false)]
+ [string]$AnyBuildClientDir,
+
+ [Parameter(mandatory=$false, ValueFromRemainingArguments=$true)]
+ [string[]]$BxlArgs
+)
+
+[Environment]::SetEnvironmentVariable("1ESSHAREDASSETS_BUILDXL_FEED_PAT", $OneEsPat, "Process")
+[Environment]::SetEnvironmentVariable("CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT", $CbPat, "Process")
+[Environment]::SetEnvironmentVariable("MSENG_GIT_PAT", $MsEngGitPat, "Process")
+[Environment]::SetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH", $NcPath, "Process")
+
+[Environment]::SetEnvironmentVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "
+{
+    'endpointCredentials': [
+        {'endpoint':'https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json', 'password':'$OneEsPat'}, 
+        {'endpoint':'https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json', 'password':'$CbPat'}
+    ]
+}", "Process")
+
+$BxlCmdArgs = @()
+
+if ($UseDev)
+{
+    $BxlCmdArgs += @("-Use", "Dev")
+}
+
+if ($DeployDev)
+{
+    $BxlCmdArgs += "-DeployDev"
+}
+
+if ($Release)
+{
+    $BxlCmdArgs += @("-DeployConfig", "Release")
+}
+
+if ($Minimal)
+{
+    $BxlCmdArgs += "-Minimal"
+}
+
+if ($EnableProcessRemoting)
+{
+    $BxlCmdArgs += "-EnableProcessRemoting"
+    $BxlCmdArgs += @("-RemoteServiceUri", $RemoteServiceUri)
+}
+
+if (-not [string]::IsNullOrEmpty($AnyBuildClientDir))
+{
+    $BxlCmdArgs += @("-AnyBuildClientDir", "$AnyBuildClientDir")
+}
+
+$BxlCmdArgs += $BxlArgs
+
+Write-Host "Call bxl.cmd $BxlCmdArgs"
+
+.\bxl.cmd $BxlCmdArgs
--- a/Shared/Scripts/bxl.ps1
+++ b/Shared/Scripts/bxl.ps1
@ -125,7 +125,7 @@ param(
    [switch]$DoNotUseDefaultCacheConfigFilePath = $false,

    [Parameter(Mandatory=$false)]
-	[switch]$UseDedupStore = $false,
+    [switch]$UseDedupStore = $false,

    [Parameter(Mandatory=$false)]
    [switch]$UseVfs = $false,
@ -139,17 +139,26 @@ param(

    [Parameter(Mandatory=$false)]
    [switch]$VsAll = $false,
-	
+
    [Parameter(Mandatory=$false)]
    [switch]$UseManagedSharedCompilation = $true,

-    [Parameter(ValueFromRemainingArguments=$true)]
-    [string[]]$DominoArguments,
-
    [Parameter(Mandatory=$false)]
    [switch]$NoQTest = $false,

-    [switch]$NoSubst = $false
+    [switch]$NoSubst = $false,
+
+    [Parameter(Mandatory=$false)]
+    [switch]$EnableProcessRemoting = $false,
+
+    [Parameter(Mandatory=$false)]
+    [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
+
+    [Parameter(Mandatory=$false)]
+    [string]$AnyBuildClientDir,
+
+    [Parameter(ValueFromRemainingArguments=$true)]
+    [string[]]$DominoArguments
 )

 $ErrorActionPreference = "Stop";
@ -211,7 +220,7 @@ $isMicrosoftInternal = [Environment]::GetEnvironmentVariable("[Sdk.BuildXL]micro
 # - this is not considered an internal build
 # We might decide to relax this once shared compilation gets enough mileage.
 if ($UseManagedSharedCompilation -and 
-        (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal))) {
+        (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal) -or $EnableProcessRemoting)) {
    $UseManagedSharedCompilation = $false
 }

@ -312,6 +321,16 @@ if (($DominoArguments -match "logsDirectory:.*").Length -eq 0 -and ($DominoArgum
    $AdditionalBuildXLArguments += "/logsToRetain:20";
 }

+if ($EnableProcessRemoting) {
+    # Unit tests are not supported for remoting because change journal is not enabled on agents
+    # and all volumes in agents have the same serial.
+    $AdditionalBuildXLArguments += @("/server-", "/enableLazyOutputs-", "/exp:lazysodeletion-", "/enableProcessRemoting+", "/processCanRunRemoteTags:compile", "/processMustRunLocalTags:telemetry:xUnit;telemetry:xUnitUntracked;telemetry:QTest", "/remoteExecutionServiceUri:$RemoteServiceUri");
+
+    if (-not [string]::IsNullOrEmpty($AnyBuildClientDir)) {
+        $AdditionalBuildXLArguments += " /p:BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR=$AnyBuildClientDir"
+    }
+}
+
 if ($Deploy -eq "LKG") {
    throw "The LKG deployment is special since it comes from a published NuGet package. It cannot be re-deployed in this selfhost wrapper.";
 }
--- a/bxlremote.cmd
+++ b/bxlremote.cmd
@ -1,31 +0,0 @@
-@ECHO OFF
-
-REM Builds using bxl.exe hooked up to AnyBuild for remoting
-REM
-REM Usage:
-REM     bxlremote.cmd <passthrough-command-line-options>
-REM where
-REM - <passthrough-command-line-options>: Additional options to pass through to bxl.cmd.
-REM
-REM Example:
-REM     bxlremote.cmd -deploy dev -minimal
-REM
-REM Prerequesites:
-REM     Requires a debug built AnyBuild repository as a sibling of the current repo root.
-
-SETLOCAL
-
-set _AnyBuildDistrib=%~dp0..\AnyBuild\distrib\Debug
-set Passthrough_Args=%*
-set _ClusterId=07F427C5-7979-415C-B6D9-01BAD5118191
-
-
-REM real cluster mode
-REM To use loopback mode, replace --RemoveExecServiceUri for --loopback
-%_AnyBuildDistrib%\AnyBuild\AnyBuild.exe --JsonConfigOverrides ProcessSubstitution.MaxParallelLocalExecutionsFactor=0 Run.DisableDirectoryMetadataDedup=true --remoteAll -v --DoNotUseMachineUtilizationForScheduling -w --NoSandboxingBuildEngine --CacheDir out\AnyBuildCache --LogDir out\AnyBuildLogs --DisableActionCache --RemoteExecServiceUri https://westus2.anybuild-test.microsoft.com/clusters/%_ClusterId%  -- bxl.cmd /enableProcessRemoting+ /enableLazyOutputs- -nosubst /maxprocmultiplier:0.75 /numRemoteLeases:100 /p:[Sdk.BuildXL]useManagedSharedCompilation=false %Passthrough_Args%
-
-if %ERRORLEVEL% NEQ 0 (
-    ENDLOCAL && EXIT /b 1
-)
-
-ENDLOCAL && EXIT /b 0
--- a/cg/nuget/cgmanifest.json
+++ b/cg/nuget/cgmanifest.json
@ -15,7 +15,7 @@
        "Type": "NuGet",
        "NuGet": {
          "Name": "AnyBuild.SDK",
-          "Version": "0.1.5"
+          "Version": "0.1.10"
        }
      }
    },
--- a/config.microsoftInternal.dsc
+++ b/config.microsoftInternal.dsc
@ -84,7 +84,7 @@ export const pkgs = isMicrosoftInternal ? [
    { id: "Microsoft.VisualStudio.Services.Governance.ComponentDetection.Contracts", version: "0.2010.80" },

    // Process remoting
-    { id: "AnyBuild.SDK", version: "0.1.5" },
+    { id: "AnyBuild.SDK", version: "0.1.10" },
 ] : [

    // Artifact packages and dependencies in OSS