diff --git a/.azdo/remoting/az-login.yml b/.azdo/remoting/az-login.yml
new file mode 100644
index 000000000..ee7a4e7e9
--- /dev/null
+++ b/.azdo/remoting/az-login.yml
@@ -0,0 +1,16 @@
+parameters:
+- name: SPAppId
+ type: string
+ default: $(BuildXLPipelinesAppId)
+- name: SPPassword
+ type: string
+ default: $(AzureApp-BuildXL-Pipelines)
+- name: SPTenant
+ type: string
+ default: $(BuildXLPipelinesTenantId)
+
+steps:
+- script: |
+ call az login --service-principal --username ${{ parameters.SPAppId }} --password ${{ parameters.SPPassword }} --tenant ${{ parameters.SPTenant }}
+ if ERRORLEVEL 1 echo "az login failed" & exit /b 1
+ displayName: 'Azure login'
diff --git a/.azdo/remoting/install-client.yml b/.azdo/remoting/install-client.yml
new file mode 100644
index 000000000..c493c6ae6
--- /dev/null
+++ b/.azdo/remoting/install-client.yml
@@ -0,0 +1,37 @@
+parameters:
+- name: AnyBuildSource
+ type: string
+ default: ''
+- name: Ring
+ type: string
+ default: 'Dogfood'
+
+steps:
+- powershell: |
+ Remove-Item -Force -Recurse "$env:LOCALAPPDATA\Microsoft\AnyBuild" -ea SilentlyContinue
+
+ # Make lowercase as AzStorage cannot handle uppercase in URL.
+ $source = "${{ parameters.AnyBuildSource }}".ToLowerInvariant()
+
+ $bootstrapperArgs = @("$source", "${{ parameters.Ring }}")
+
+ Write-Host "Bootstrapper args: '$bootstrapperArgs'";
+
+ while ($true)
+ {
+ $script = ((curl.exe -s -S --retry 10 --retry-connrefused "$source/bootstrapper.ps1") | Out-String)
+ Write-Host "Downloaded script: `r`n $script";
+ if ($LASTEXITCODE -eq 0)
+ {
+ break;
+ }
+
+ # We sometimes get a 404 error when downloading the bootstrapper
+ # while the bootstrapper script is updating. Needs an eventual fix instead of this workaround.
+ Write-Host "ERROR: Failed downloading the bootstrapper script. Trying again."
+ }
+
+ Invoke-Command -ScriptBlock ([scriptblock]::Create($script)) -ArgumentList $bootstrapperArgs
+ failOnStderr: true
+ displayName: 'Install AnyBuild client'
+ continueOnError: false
\ No newline at end of file
diff --git a/.azdo/remoting/pipeline.yml b/.azdo/remoting/pipeline.yml
new file mode 100644
index 000000000..7abb8c1e9
--- /dev/null
+++ b/.azdo/remoting/pipeline.yml
@@ -0,0 +1,85 @@
+trigger: none
+
+pool:
+ name: BuildXL-DevOpsAgents-PME
+
+variables:
+ NugetCredentialProviderPath: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\bin\Debug\netcoreapp2.1\win-x64'
+ Datacenter: 'westus2'
+ ClusterName: 'Bxl1'
+ RemoteUri: 'https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191'
+ ClientContainerUrl: 'https://anybuild$(ClusterName)$(Datacenter).blob.core.windows.net/clientreleases'
+ BootstrapArgs: '-DeployDev /p:[Sdk.BuildXL]microsoftInternal=1'
+ RemotingArgs: '-UseDev -Minimal -EnableProcessRemoting -RemoteServiceUri $(RemoteUri) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID=$(BuildXLPipelinesAppId) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV=AnyBuildServicePrincipalPwd /numRemoteAgentLeases:24 /remotingThresholdMultiplier:1.25 /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=*'
+
+steps:
+
+- checkout: self
+ fetchDepth: 1
+
+- template: ./az-login.yml
+
+- script: 'C:\windows\system32\fsutil usn createjournal m=0x20000000 a=0x8000000 D:'
+ displayName: 'Enable Journaling for D drive'
+
+- script: |
+ if EXIST %AppData%\NuGet\NuGet.Config (
+ del %AppData%\NuGet\NuGet.Config
+ )
+ displayName: 'Nuget Hack: delete old nuget.config first'
+
+- task: UseDotNet@2
+ displayName: 'Use .NET Core sdk 6.x'
+ inputs:
+ version: 6.x
+
+- task: DotNetCoreCLI@2
+ displayName: 'Build BuildXL.Tools.CredentialProvider'
+ inputs:
+ projects: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\BuildXL.Tools.CredentialProvider.csproj'
+ arguments: '-r win-x64'
+
+- template: ./install-client.yml
+ parameters:
+ AnyBuildSource: $(ClientContainerUrl)
+
+- task: BatchScript@1
+ displayName: 'Kill existing BuildXL processes'
+ inputs:
+ filename: Shared/Scripts/KillBxlInstancesInRepo.cmd
+ continueOnError: true
+ condition: always()
+
+- task: PowerShell@2
+ displayName: 'Build BXL'
+ inputs:
+ targetType: filePath
+ filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
+ arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(BootstrapArgs)'
+
+- task: PowerShell@2
+ displayName: 'Run BXL with remoting'
+ inputs:
+ targetType: filePath
+ filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
+ arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(RemotingArgs)'
+ env:
+ AnyBuildServicePrincipalPwd: $(AzureApp-BuildXL-Pipelines)
+
+- task: BatchScript@1
+ displayName: 'Kill existing BuildXL processes'
+ inputs:
+ filename: Shared/Scripts/KillBxlInstancesInRepo.cmd
+ continueOnError: true
+ condition: always()
+
+- task: PublishPipelineArtifact@1
+ displayName: 'Upload logs'
+ inputs:
+ targetPath: '$(Build.SourcesDirectory)\Out\Logs'
+ condition: always()
+
+- template: ./validate.yml
+ parameters:
+ LogDirectory: '$(Build.SourcesDirectory)\Out\Logs'
+
diff --git a/.azdo/remoting/validate.yml b/.azdo/remoting/validate.yml
new file mode 100644
index 000000000..917eadf35
--- /dev/null
+++ b/.azdo/remoting/validate.yml
@@ -0,0 +1,32 @@
+parameters:
+- name: LogDirectory
+ type: string
+ default: '$(Build.SourcesDirectory)\Out\Logs'
+
+steps:
+- powershell: |
+ $f = Get-ChildItem -Path '${{ parameters.LogDirectory }}\*\AnyBuildDaemonManager.log' -Recurse | Select-Object -First 1
+ $bxlWithRemotingLogDir = $f.DirectoryName
+ $statsContent = Get-Content -Path (Join-Path $bxlWithRemotingLogDir "BuildXL.stats") | Out-String
+ $stats = ConvertFrom-StringData -StringData $statsContent
+ $runRemoteCount = $stats.Get_Item("PipExecution.TotalRunRemoteProcesses")
+ $runLocalFallbackCount = $stats.Get_Item("PipExecution.TotalRemoteFallbackRetries")
+
+ Write-Host "Run remote: $runRemoteCount"
+ Write-Host "Run local due to fallback: $runLocalFallbackCount"
+
+ if ($runRemoteCount -eq 0)
+ {
+ Write-Host "No process pip ran remotely"
+ exit 1
+ }
+
+ if ($runLocalFallbackCount -ne 0)
+ {
+ Write-Host "$runLocalFallbackCount remoted process pip(s) ran locally"
+ exit 1
+ }
+
+ exit 0
+ continueOnError: true
+ displayName: 'Validate remoting'
\ No newline at end of file
diff --git a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs
index c51d9bf42..8f4156009 100644
--- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs
+++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessManager.cs
@@ -118,7 +118,8 @@ namespace BuildXL.Processes.Remoting
closeDaemonOnDispose: true,
m_executionContext.CancellationToken,
logDirectory: logDir,
- additionalAnyBuildParameters: extraParams);
+ additionalAnyBuildParameters: extraParams,
+ inheritHandlesOnProcessCreation: false);
}
catch (Exception e)
{
@@ -165,6 +166,20 @@ namespace BuildXL.Processes.Remoting
$"--CacheDir {localCacheDir}",
};
+ if (!string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId))
+ {
+ args.Add("--ClientApplicationId");
+ args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId);
+
+ if (string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv))
+ {
+ throw new BuildXLException($"Service principal password is required for starting AnyBuild daemon");
+ }
+
+ args.Add("--ClientSecretEnvironmentVariable");
+ args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv);
+ }
+
if (!string.IsNullOrEmpty(m_configuration.Schedule.RemoteExecutionServiceUri))
{
try
diff --git a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs
index f44222573..3d63dc60f 100644
--- a/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs
+++ b/Public/Src/Engine/Processes/Remoting/AnyBuild/AnyBuildRemoteProcessPipResult.cs
@@ -41,9 +41,8 @@ namespace BuildXL.Processes.Remoting.AnyBuild
ExitCode = result.ExitCode,
StdOut = result.StdOut,
StdErr = result.StdErr,
- Disposition = result.Disposition switch
+ Disposition = result.ExecutionDisposition switch
{
- CommandExecutionDisposition.Unknown => RemoteResultDisposition.Unknown,
CommandExecutionDisposition.CacheHit => RemoteResultDisposition.CacheHit,
CommandExecutionDisposition.Remoted => RemoteResultDisposition.Remoted,
CommandExecutionDisposition.RanLocally => RemoteResultDisposition.RanLocally,
diff --git a/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs b/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs
index 41bb9111e..7936a55a3 100644
--- a/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs
+++ b/Public/Src/Engine/Processes/Remoting/RemoteResultDisposition.cs
@@ -8,11 +8,6 @@ namespace BuildXL.Processes.Remoting
///
public enum RemoteResultDisposition
{
- ///
- /// Default zero value.
- ///
- Unknown,
-
///
/// Process completed with cache hit.
///
diff --git a/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs b/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
index 04b9f4f24..db5b84ab8 100644
--- a/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
+++ b/Public/Src/Utilities/Configuration/EngineEnvironmentSettings.cs
@@ -88,6 +88,16 @@ namespace BuildXL.Utilities.Configuration
///
public static readonly Setting AnyBuildExtraArgs = CreateSetting("BUILDXL_ANYBUILD_EXTRA_ARGS", value => value);
+ ///
+ /// Service principal app id for starting AnyBuild daemon.
+ ///
+ public static readonly Setting AnyBuildServicePrincipalAppId = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID", value => value);
+
+ ///
+ /// Environment variable where the service principal password for starting AnyBuild daemon is stored.
+ ///
+ public static readonly Setting AnyBuildServicePrincipalPwdEnv = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV", value => value);
+
///
/// Indicates whether the application should fail fast on null reference exceptions
///
diff --git a/RunBxlWithPAT.ps1 b/RunBxlWithPAT.ps1
new file mode 100644
index 000000000..10dc3805d
--- /dev/null
+++ b/RunBxlWithPAT.ps1
@@ -0,0 +1,82 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [Parameter(mandatory=$true)]
+ [String]$OneEsPat,
+ [Parameter(mandatory=$true)]
+ [String]$CbPat,
+ [Parameter(mandatory=$true)]
+ [String]$NcPath,
+ [Parameter(mandatory=$true)]
+ [String]$MsEngGitPat,
+
+ [Parameter(Mandatory=$false)]
+ [switch]$DeployDev = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$UseDev = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$Minimal = $false,
+ [Parameter(Mandatory=$false)]
+ [switch]$Release = $false,
+
+ [Parameter(Mandatory=$false)]
+ [switch]$EnableProcessRemoting = $false,
+ [Parameter(Mandatory=$false)]
+ [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
+ [Parameter(Mandatory=$false)]
+ [string]$AnyBuildClientDir,
+
+ [Parameter(mandatory=$false, ValueFromRemainingArguments=$true)]
+ [string[]]$BxlArgs
+)
+
+[Environment]::SetEnvironmentVariable("1ESSHAREDASSETS_BUILDXL_FEED_PAT", $OneEsPat, "Process")
+[Environment]::SetEnvironmentVariable("CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT", $CbPat, "Process")
+[Environment]::SetEnvironmentVariable("MSENG_GIT_PAT", $MsEngGitPat, "Process")
+[Environment]::SetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH", $NcPath, "Process")
+
+[Environment]::SetEnvironmentVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "
+{
+ 'endpointCredentials': [
+ {'endpoint':'https://pkgs.dev.azure.com/1essharedassets/_packaging/BuildXL/nuget/v3/index.json', 'password':'$OneEsPat'},
+ {'endpoint':'https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json', 'password':'$CbPat'}
+ ]
+}", "Process")
+
+$BxlCmdArgs = @()
+
+if ($UseDev)
+{
+ $BxlCmdArgs += @("-Use", "Dev")
+}
+
+if ($DeployDev)
+{
+ $BxlCmdArgs += "-DeployDev"
+}
+
+if ($Release)
+{
+ $BxlCmdArgs += @("-DeployConfig", "Release")
+}
+
+if ($Minimal)
+{
+ $BxlCmdArgs += "-Minimal"
+}
+
+if ($EnableProcessRemoting)
+{
+ $BxlCmdArgs += "-EnableProcessRemoting"
+ $BxlCmdArgs += @("-RemoteServiceUri", $RemoteServiceUri)
+}
+
+if (-not [string]::IsNullOrEmpty($AnyBuildClientDir))
+{
+ $BxlCmdArgs += @("-AnyBuildClientDir", "$AnyBuildClientDir")
+}
+
+$BxlCmdArgs += $BxlArgs
+
+Write-Host "Call bxl.cmd $BxlCmdArgs"
+
+.\bxl.cmd $BxlCmdArgs
\ No newline at end of file
diff --git a/Shared/Scripts/bxl.ps1 b/Shared/Scripts/bxl.ps1
index e00140bad..c2cdc0b77 100644
--- a/Shared/Scripts/bxl.ps1
+++ b/Shared/Scripts/bxl.ps1
@@ -125,7 +125,7 @@ param(
[switch]$DoNotUseDefaultCacheConfigFilePath = $false,
[Parameter(Mandatory=$false)]
- [switch]$UseDedupStore = $false,
+ [switch]$UseDedupStore = $false,
[Parameter(Mandatory=$false)]
[switch]$UseVfs = $false,
@@ -139,17 +139,26 @@ param(
[Parameter(Mandatory=$false)]
[switch]$VsAll = $false,
-
+
[Parameter(Mandatory=$false)]
[switch]$UseManagedSharedCompilation = $true,
- [Parameter(ValueFromRemainingArguments=$true)]
- [string[]]$DominoArguments,
-
[Parameter(Mandatory=$false)]
[switch]$NoQTest = $false,
- [switch]$NoSubst = $false
+ [switch]$NoSubst = $false,
+
+ [Parameter(Mandatory=$false)]
+ [switch]$EnableProcessRemoting = $false,
+
+ [Parameter(Mandatory=$false)]
+ [string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
+
+ [Parameter(Mandatory=$false)]
+ [string]$AnyBuildClientDir,
+
+ [Parameter(ValueFromRemainingArguments=$true)]
+ [string[]]$DominoArguments
)
$ErrorActionPreference = "Stop";
@@ -211,7 +220,7 @@ $isMicrosoftInternal = [Environment]::GetEnvironmentVariable("[Sdk.BuildXL]micro
# - this is not considered an internal build
# We might decide to relax this once shared compilation gets enough mileage.
if ($UseManagedSharedCompilation -and
- (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal))) {
+ (($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal) -or $EnableProcessRemoting)) {
$UseManagedSharedCompilation = $false
}
@@ -312,6 +321,16 @@ if (($DominoArguments -match "logsDirectory:.*").Length -eq 0 -and ($DominoArgum
$AdditionalBuildXLArguments += "/logsToRetain:20";
}
+if ($EnableProcessRemoting) {
+ # Unit tests are not supported for remoting because change journal is not enabled on agents
+ # and all volumes in agents have the same serial.
+ $AdditionalBuildXLArguments += @("/server-", "/enableLazyOutputs-", "/exp:lazysodeletion-", "/enableProcessRemoting+", "/processCanRunRemoteTags:compile", "/processMustRunLocalTags:telemetry:xUnit;telemetry:xUnitUntracked;telemetry:QTest", "/remoteExecutionServiceUri:$RemoteServiceUri");
+
+ if (-not [string]::IsNullOrEmpty($AnyBuildClientDir)) {
+ $AdditionalBuildXLArguments += " /p:BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR=$AnyBuildClientDir"
+ }
+}
+
if ($Deploy -eq "LKG") {
throw "The LKG deployment is special since it comes from a published NuGet package. It cannot be re-deployed in this selfhost wrapper.";
}
diff --git a/bxlremote.cmd b/bxlremote.cmd
deleted file mode 100644
index 0bc0ab3a6..000000000
--- a/bxlremote.cmd
+++ /dev/null
@@ -1,31 +0,0 @@
-@ECHO OFF
-
-REM Builds using bxl.exe hooked up to AnyBuild for remoting
-REM
-REM Usage:
-REM bxlremote.cmd
-REM where
-REM - : Additional options to pass through to bxl.cmd.
-REM
-REM Example:
-REM bxlremote.cmd -deploy dev -minimal
-REM
-REM Prerequesites:
-REM Requires a debug built AnyBuild repository as a sibling of the current repo root.
-
-SETLOCAL
-
-set _AnyBuildDistrib=%~dp0..\AnyBuild\distrib\Debug
-set Passthrough_Args=%*
-set _ClusterId=07F427C5-7979-415C-B6D9-01BAD5118191
-
-
-REM real cluster mode
-REM To use loopback mode, replace --RemoveExecServiceUri for --loopback
-%_AnyBuildDistrib%\AnyBuild\AnyBuild.exe --JsonConfigOverrides ProcessSubstitution.MaxParallelLocalExecutionsFactor=0 Run.DisableDirectoryMetadataDedup=true --remoteAll -v --DoNotUseMachineUtilizationForScheduling -w --NoSandboxingBuildEngine --CacheDir out\AnyBuildCache --LogDir out\AnyBuildLogs --DisableActionCache --RemoteExecServiceUri https://westus2.anybuild-test.microsoft.com/clusters/%_ClusterId% -- bxl.cmd /enableProcessRemoting+ /enableLazyOutputs- -nosubst /maxprocmultiplier:0.75 /numRemoteLeases:100 /p:[Sdk.BuildXL]useManagedSharedCompilation=false %Passthrough_Args%
-
-if %ERRORLEVEL% NEQ 0 (
- ENDLOCAL && EXIT /b 1
-)
-
-ENDLOCAL && EXIT /b 0
\ No newline at end of file
diff --git a/cg/nuget/cgmanifest.json b/cg/nuget/cgmanifest.json
index 66f0ad9a6..9d3da5c65 100644
--- a/cg/nuget/cgmanifest.json
+++ b/cg/nuget/cgmanifest.json
@@ -15,7 +15,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "AnyBuild.SDK",
- "Version": "0.1.5"
+ "Version": "0.1.10"
}
}
},
diff --git a/config.microsoftInternal.dsc b/config.microsoftInternal.dsc
index fb7fe485a..dce7bd406 100644
--- a/config.microsoftInternal.dsc
+++ b/config.microsoftInternal.dsc
@@ -84,7 +84,7 @@ export const pkgs = isMicrosoftInternal ? [
{ id: "Microsoft.VisualStudio.Services.Governance.ComponentDetection.Contracts", version: "0.2010.80" },
// Process remoting
- { id: "AnyBuild.SDK", version: "0.1.5" },
+ { id: "AnyBuild.SDK", version: "0.1.10" },
] : [
// Artifact packages and dependencies in OSS