зеркало из https://github.com/microsoft/BuildXL.git
Merged PR 648438: Developer experience for BuildXL-selfhost remoting
Changes need for developer experience for BuildXL-selfhost remoting: * No need to install AnyBuild * Use AnyBuild.json for repo configuration * Simplify bxl.ps1 options * Remove redundant engine environment settings **TODO** - [X] Use prod cluster - [ ] Selectively include test pips (e.g., cache tests) to can-be-remoted pips - [ ] Remove `/server-` - [ ] Enable shared compilation More pips will be added once it's proven successfully in the pipeline. /server- currently cannot be removed (see details in bxl.ps1). shared compilation still needs more investigation. Related work items: #1915141
This commit is contained in:
Iman Narasamdya
Родитель
b1f7ca7b66
Коммит
dd1f690efd
|
|
@ -1,37 +0,0 @@
|
|||
parameters:
|
||||
- name: AnyBuildSource
|
||||
type: string
|
||||
default: ''
|
||||
- name: Ring
|
||||
type: string
|
||||
default: 'Dogfood'
|
||||
|
||||
steps:
|
||||
- powershell: |
|
||||
Remove-Item -Force -Recurse "$env:LOCALAPPDATA\Microsoft\AnyBuild" -ea SilentlyContinue
|
||||
|
||||
# Make lowercase as AzStorage cannot handle uppercase in URL.
|
||||
$source = "${{ parameters.AnyBuildSource }}".ToLowerInvariant()
|
||||
|
||||
$bootstrapperArgs = @("$source", "${{ parameters.Ring }}")
|
||||
|
||||
Write-Host "Bootstrapper args: '$bootstrapperArgs'";
|
||||
|
||||
while ($true)
|
||||
{
|
||||
$script = ((curl.exe -s -S --retry 10 --retry-connrefused "$source/bootstrapper.ps1") | Out-String)
|
||||
Write-Host "Downloaded script: `r`n $script";
|
||||
if ($LASTEXITCODE -eq 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
# We sometimes get a 404 error when downloading the bootstrapper
|
||||
# while the bootstrapper script is updating. Needs an eventual fix instead of this workaround.
|
||||
Write-Host "ERROR: Failed downloading the bootstrapper script. Trying again."
|
||||
}
|
||||
|
||||
Invoke-Command -ScriptBlock ([scriptblock]::Create($script)) -ArgumentList $bootstrapperArgs
|
||||
failOnStderr: true
|
||||
displayName: 'Install AnyBuild client'
|
||||
continueOnError: false
|
||||
|
|
@ -1,3 +1,15 @@
|
|||
parameters:
|
||||
# - name: AnyBuildSource
|
||||
# type: string
|
||||
# default: 'https://anybuildbxl1westus2.blob.core.windows.net/clientreleases'
|
||||
# default: 'https://anybuild.azureedge.net/clientreleases'
|
||||
- name: BuildXLArguments
|
||||
type: string
|
||||
default: '/numRemoteAgentLeases:24 /remotingThresholdMultiplier:1.25 /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=*'
|
||||
- name: AnyBuildArguments
|
||||
type: string
|
||||
default: '--NoCheckForUpdates'
|
||||
|
||||
trigger: none
|
||||
|
||||
pool:
|
||||
|
|
@ -5,12 +17,20 @@ pool:
|
|||
|
||||
variables:
|
||||
NugetCredentialProviderPath: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\bin\Debug\netcoreapp2.1\win-x64'
|
||||
Datacenter: 'westus2'
|
||||
ClusterName: 'Bxl1'
|
||||
RemoteUri: 'https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191'
|
||||
ClientContainerUrl: 'https://anybuild$(ClusterName)$(Datacenter).blob.core.windows.net/clientreleases'
|
||||
BootstrapArgs: '-DeployDev /p:[Sdk.BuildXL]microsoftInternal=1'
|
||||
RemotingArgs: '-UseDev -Minimal -EnableProcessRemoting -RemoteServiceUri $(RemoteUri) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID=$(BuildXLPipelinesAppId) /p:BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV=AnyBuildServicePrincipalPwd /numRemoteAgentLeases:24 /remotingThresholdMultiplier:1.25 /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=*'
|
||||
# AnyBuild.json
|
||||
# {
|
||||
# "DefaultClusterUri": "https://westus2.anybuild.microsoft.com/clusters/5cff1ffe-c438-4382-be19-140c0cd893ae",
|
||||
# "DefaultClusterName": "BuildXL1"
|
||||
# }
|
||||
# AnyBuild.json
|
||||
# {
|
||||
# "DefaultClusterUri": "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
|
||||
# "DefaultClusterName": "Bxl1"
|
||||
# }
|
||||
CommonArgs: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode)'
|
||||
BootstrapArgs: '-Minimal -Deploy Dev /p:[Sdk.BuildXL]microsoftInternal=1'
|
||||
AnyBuildAuthArgs: '--ClientApplicationId~~$(BuildXLPipelinesAppId)~~--ClientSecretEnvironmentVariable~~AnyBuildServicePrincipalPasswd'
|
||||
RemotingArgs: '-Use Dev -Minimal -EnableProcessRemoting /p:BUILDXL_ANYBUILD_EXTRA_ARGS=$(AnyBuildAuthArgs)~~${{ parameters.AnyBuildArguments }} ${{ parameters.BuildXLArguments }}'
|
||||
|
||||
steps:
|
||||
|
||||
|
|
@ -39,9 +59,11 @@ steps:
|
|||
projects: '$(Build.SourcesDirectory)\Public\Src\Tools\BuildXL.Tools.CredentialProvider\BuildXL.Tools.CredentialProvider.csproj'
|
||||
arguments: '-r win-x64'
|
||||
|
||||
- template: ./install-client.yml
|
||||
parameters:
|
||||
AnyBuildSource: $(ClientContainerUrl)
|
||||
- powershell: |
|
||||
Remove-Item -Force -Recurse "$env:LOCALAPPDATA\Microsoft\AnyBuild" -ea SilentlyContinue
|
||||
failOnStderr: true
|
||||
displayName: 'Remove existing AnyBuild client'
|
||||
continueOnError: false
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'Kill existing BuildXL processes'
|
||||
|
|
@ -51,20 +73,20 @@ steps:
|
|||
condition: always()
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Build BXL'
|
||||
displayName: 'Build BXL (bootstrap)'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
|
||||
arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(BootstrapArgs)'
|
||||
arguments: '$(CommonArgs) $(BootstrapArgs)'
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Run BXL with remoting'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: '$(Build.SourcesDirectory)\RunBxlWithPAT.ps1'
|
||||
arguments: '-OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -NcPath $(NugetCredentialProviderPath) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) $(RemotingArgs)'
|
||||
arguments: '$(CommonArgs) $(RemotingArgs)'
|
||||
env:
|
||||
AnyBuildServicePrincipalPwd: $(AzureApp-BuildXL-Pipelines)
|
||||
AnyBuildServicePrincipalPasswd: $(AzureApp-BuildXL-Pipelines)
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'Kill existing BuildXL processes'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"DefaultClusterUri": "https://westus2.anybuild.microsoft.com/clusters/5cff1ffe-c438-4382-be19-140c0cd893ae",
|
||||
"DefaultClusterName": "BuildXL1"
|
||||
}
|
||||
|
|
@ -353,28 +353,6 @@ namespace BuildXL
|
|||
"diagnostic",
|
||||
"diag",
|
||||
opt => loggingConfiguration.Diagnostic |= CommandLineUtilities.ParseEnumOption<DiagnosticLevels>(opt)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"dumpFailedPips",
|
||||
opt => loggingConfiguration.DumpFailedPips = opt),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"dumpFailedPipsLogLimit",
|
||||
opt => loggingConfiguration.DumpFailedPipsLogLimit = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"dumpFailedPipsWithDynamicData",
|
||||
opt => loggingConfiguration.DumpFailedPipsWithDynamicData = opt),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"earlyWorkerRelease",
|
||||
sign => distributionConfiguration.EarlyWorkerRelease = sign),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"earlyWorkerReleaseMultiplier",
|
||||
opt =>
|
||||
distributionConfiguration.EarlyWorkerReleaseMultiplier = CommandLineUtilities.ParseDoubleOption(opt, 0, 5)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"enforceAccessPoliciesOnDirectoryCreation",
|
||||
sign => sandboxConfiguration.EnforceAccessPoliciesOnDirectoryCreation = sign),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"emitSpotlightIndexingWarning",
|
||||
sign => layoutConfiguration.EmitSpotlightIndexingWarning = sign),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"disableConHostSharing",
|
||||
sign => engineConfiguration.DisableConHostSharing = sign),
|
||||
|
|
@ -409,9 +387,28 @@ namespace BuildXL
|
|||
"distributedBuildWorker",
|
||||
"dbw",
|
||||
opt => ParseServiceLocation(opt, distributionConfiguration.BuildWorkers)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"dumpFailedPips",
|
||||
opt => loggingConfiguration.DumpFailedPips = opt),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"dumpFailedPipsLogLimit",
|
||||
opt => loggingConfiguration.DumpFailedPipsLogLimit = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"dumpFailedPipsWithDynamicData",
|
||||
opt => loggingConfiguration.DumpFailedPipsWithDynamicData = opt),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"earlyWorkerRelease",
|
||||
sign => distributionConfiguration.EarlyWorkerRelease = sign),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"earlyWorkerReleaseMultiplier",
|
||||
opt =>
|
||||
distributionConfiguration.EarlyWorkerReleaseMultiplier = CommandLineUtilities.ParseDoubleOption(opt, 0, 5)),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"elideMinimalGraphEnumerationAbsentPathProbes",
|
||||
sign => cacheConfiguration.ElideMinimalGraphEnumerationAbsentPathProbes = sign),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"emitSpotlightIndexingWarning",
|
||||
sign => layoutConfiguration.EmitSpotlightIndexingWarning = sign),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"enableAsyncLogging",
|
||||
sign => loggingConfiguration.EnableAsyncLogging = sign),
|
||||
|
|
@ -421,14 +418,9 @@ namespace BuildXL
|
|||
OptionHandlerFactory.CreateBoolOption(
|
||||
"enableHistoricCommitMemoryProjection",
|
||||
sign => schedulingConfiguration.EnableHistoricCommitMemoryProjection = sign),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"hashType",
|
||||
option =>
|
||||
{
|
||||
var hashType = option.Value.FindHashTypeByName();
|
||||
ContentHashingUtilities.SetDefaultHashType(hashType);
|
||||
cacheConfiguration.UseDedupStore = hashType.IsValidDedup();
|
||||
}),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"enforceAccessPoliciesOnDirectoryCreation",
|
||||
sign => sandboxConfiguration.EnforceAccessPoliciesOnDirectoryCreation = sign),
|
||||
OptionHandlerFactory.CreateBoolOption(
|
||||
"enableGrpc",
|
||||
sign =>
|
||||
|
|
@ -557,6 +549,14 @@ namespace BuildXL
|
|||
OptionHandlerFactory.CreateBoolOption(
|
||||
"hardExitOnErrorInDetours",
|
||||
sign => sandboxConfiguration.HardExitOnErrorInDetours = sign),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"hashType",
|
||||
option =>
|
||||
{
|
||||
var hashType = option.Value.FindHashTypeByName();
|
||||
ContentHashingUtilities.SetDefaultHashType(hashType);
|
||||
cacheConfiguration.UseDedupStore = hashType.IsValidDedup();
|
||||
}),
|
||||
#if PLATFORM_OSX
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"numberOfKextConnections", // TODO: deprecate and remove
|
||||
|
|
@ -912,9 +912,6 @@ namespace BuildXL
|
|||
OptionHandlerFactory.CreateOption(
|
||||
"relatedActivityId",
|
||||
opt => loggingConfiguration.RelatedActivityId = CommandLineUtilities.ParseStringOption(opt)),
|
||||
OptionHandlerFactory.CreateOption(
|
||||
"remoteExecutionServiceUri",
|
||||
opt => schedulingConfiguration.RemoteExecutionServiceUri = CommandLineUtilities.ParseStringOption(opt)),
|
||||
OptionHandlerFactory.CreateBoolOptionWithValue(
|
||||
"remoteTelemetry",
|
||||
(opt, sign) =>
|
||||
|
|
|
|||
|
|
@ -938,11 +938,6 @@ namespace BuildXL
|
|||
Strings.HelpText_DisplayHelp_NumRemoteAgentLeases,
|
||||
HelpLevel.Verbose);
|
||||
|
||||
hw.WriteOption(
|
||||
"/remoteExecutionServiceUri:<uri>",
|
||||
Strings.HelpText_DisplayHelp_RemoteExecutionServiceUri,
|
||||
HelpLevel.Verbose);
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1105,9 +1105,6 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
|
|||
<data name="HelpText_DisplayHelp_RemotingThresholdMultiplier" xml:space="preserve">
|
||||
<value>Multiplier for threshold before starting to remote process pips when /enableProcessRemoting is set to true. The threshold is obtained by multiplying /maxProc with this multiplier. Defaults to 1.5.</value>
|
||||
</data>
|
||||
<data name="HelpText_DisplayHelp_RemoteExecutionServiceUri" xml:space="preserve">
|
||||
<value>URI of cluster to use for remoting processes. Applicable only when /enableProcessRemoting is set to true</value>
|
||||
</data>
|
||||
<data name="HelpText_DisplayHelp_RunInSubst" xml:space="preserve">
|
||||
<value>Improves path stability across potentially heterogeneous machines by internally mapping a source path (typically the source of the repo to build) into a drive letter. If the source path is not explicitly provided with /substSource, the location of the main config file is used. Only effective on Windows, in other platforms the option is ignored. Useful for dev cache.</value>
|
||||
</data>
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ namespace Processes {
|
|||
...addIf(BuildXLSdk.isFullFramework,
|
||||
BuildXLSdk.NetFx.System.IO.Compression.dll,
|
||||
BuildXLSdk.NetFx.System.Management.dll,
|
||||
BuildXLSdk.NetFx.System.Net.Http.dll,
|
||||
NetFx.Netstandard.dll
|
||||
),
|
||||
...addIf(BuildXLSdk.isDotNetCoreBuild,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,274 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.Configuration;
|
||||
using BuildXL.Utilities.Instrumentation.Common;
|
||||
using BuildXL.Utilities.Tasks;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace BuildXL.Processes.Remoting
|
||||
{
|
||||
/// <summary>
|
||||
/// AnyBuild client installer.
|
||||
/// </summary>
|
||||
public class AnyBuildInstaller : IRemoteProcessManagerInstaller
|
||||
{
|
||||
/// <summary>
|
||||
/// Installation rings.
|
||||
/// </summary>
|
||||
public enum Ring
|
||||
{
|
||||
/// <summary>
|
||||
/// Dogfood.
|
||||
/// </summary>
|
||||
Dogfood,
|
||||
|
||||
/// <summary>
|
||||
/// Production.
|
||||
/// </summary>
|
||||
Production
|
||||
}
|
||||
|
||||
private const string DefaultAnyBuildClientInstallSource = "https://anybuild.azureedge.net/clientreleases";
|
||||
|
||||
private readonly string m_source;
|
||||
private readonly Ring m_ring;
|
||||
private readonly LoggingContext m_loggingContext;
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of <see cref="AnyBuildInstaller"/>.
|
||||
/// </summary>
|
||||
/// <param name="source">Source of AnyBuild client.</param>
|
||||
/// <param name="ring">Ring.</param>
|
||||
/// <param name="loggingContext">Logging context.</param>
|
||||
public AnyBuildInstaller(string source, Ring ring, LoggingContext loggingContext)
|
||||
{
|
||||
m_source = source;
|
||||
m_ring = ring;
|
||||
m_loggingContext = loggingContext;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of <see cref="AnyBuildInstaller"/>.
|
||||
/// </summary>
|
||||
/// <param name="sourceAndRing">Source and ring of AnyBuild client.</param>
|
||||
/// <param name="loggingContext">Logging context.</param>
|
||||
public AnyBuildInstaller(string sourceAndRing, LoggingContext loggingContext)
|
||||
{
|
||||
(m_source, m_ring) = ParseSourceAndRing(sourceAndRing);
|
||||
m_loggingContext = loggingContext;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of <see cref="AnyBuildInstaller"/>.
|
||||
/// </summary>
|
||||
/// <param name="loggingContext">Logging context.</param>
|
||||
public AnyBuildInstaller(LoggingContext loggingContext)
|
||||
: this(EngineEnvironmentSettings.AnyBuildClientSource.Value ?? DefaultAnyBuildClientInstallSource, loggingContext)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses source and ring in URI.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// An example of <paramref name="sourceAndRing"/> is 'https://anybuild.azureedge.net/clientreleases?ring=Dogfood'
|
||||
/// </remarks>
|
||||
private static (string, Ring) ParseSourceAndRing(string sourceAndRing)
|
||||
{
|
||||
int indexOfQuestionMark = sourceAndRing.LastIndexOf('?');
|
||||
if (indexOfQuestionMark == -1)
|
||||
{
|
||||
return (sourceAndRing, Ring.Production);
|
||||
}
|
||||
|
||||
string source = sourceAndRing.Substring(0, indexOfQuestionMark);
|
||||
|
||||
// Use Production ring as default.
|
||||
Ring ring = Ring.Production;
|
||||
|
||||
if ((indexOfQuestionMark + 1) >= sourceAndRing.Length)
|
||||
{
|
||||
return (source, ring);
|
||||
}
|
||||
|
||||
string[] ringKvp = sourceAndRing.Substring(indexOfQuestionMark + 1).Split('=');
|
||||
if (ringKvp.Length != 2 || !string.Equals(ringKvp[0], "ring", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return (source, ring);
|
||||
}
|
||||
|
||||
if (!Enum.TryParse(ringKvp[1], true, out Ring parsedRing))
|
||||
{
|
||||
return (source, ring);
|
||||
}
|
||||
|
||||
return (source, parsedRing);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Installs AnyBuild client.
|
||||
/// </summary>
|
||||
/// <param name="token">Cancellation token.</param>
|
||||
/// <param name="forceInstall">Force installation by cleaning existing installation when set to true.</param>
|
||||
public Task<bool> InstallAsync(CancellationToken token, bool forceInstall = false)
|
||||
{
|
||||
if (EngineEnvironmentSettings.AnyBuildInstallDir.Value != null)
|
||||
{
|
||||
// Do not perform installation if user specifies a particular AnyBuild installation.
|
||||
return BoolTask.True;
|
||||
}
|
||||
|
||||
if (OperatingSystemHelper.IsWindowsOS)
|
||||
{
|
||||
return InstallWinAsync(token, forceInstall);
|
||||
}
|
||||
|
||||
throw new NotSupportedException($"{nameof(AnyBuildInstaller)} does not support {Environment.OSVersion.Platform}");
|
||||
}
|
||||
|
||||
private async Task<bool> InstallWinAsync(CancellationToken token, bool forceInstall = false)
|
||||
{
|
||||
string abDir = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
"Microsoft",
|
||||
"AnyBuild");
|
||||
|
||||
string abCmd = Path.Combine(abDir, "AnyBuild.cmd");
|
||||
|
||||
forceInstall |= EngineEnvironmentSettings.AnyBuildForceInstall.Value;
|
||||
|
||||
if (forceInstall && Directory.Exists(abDir))
|
||||
{
|
||||
Directory.Delete(abDir, true);
|
||||
}
|
||||
|
||||
if (File.Exists(abCmd))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
Tracing.Logger.Log.InstallAnyBuildClient(m_loggingContext, m_source, m_ring.ToString());
|
||||
|
||||
string? bootstrapperFile = await DownloadBootstrapperWinAsync(token);
|
||||
|
||||
if (bootstrapperFile == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return await ExecuteBootstrapper(bootstrapperFile);
|
||||
}
|
||||
|
||||
private async Task<bool> ExecuteBootstrapper(string bootstrapperFile)
|
||||
{
|
||||
var output = new StringBuilder(512);
|
||||
var error = new StringBuilder(512);
|
||||
|
||||
var process = new Process()
|
||||
{
|
||||
StartInfo = new ProcessStartInfo(
|
||||
"powershell.exe",
|
||||
$"-NoProfile -ExecutionPolicy ByPass -File \"{bootstrapperFile}\" {m_source} {m_ring}")
|
||||
{
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
ErrorDialog = false,
|
||||
WindowStyle = ProcessWindowStyle.Hidden,
|
||||
},
|
||||
EnableRaisingEvents = false,
|
||||
};
|
||||
|
||||
process.OutputDataReceived += (_, eventArgs) =>
|
||||
{
|
||||
if (eventArgs.Data is not null)
|
||||
{
|
||||
output.AppendLine(eventArgs.Data);
|
||||
}
|
||||
};
|
||||
|
||||
process.ErrorDataReceived += (_, eventArgs) =>
|
||||
{
|
||||
if (eventArgs.Data is not null)
|
||||
{
|
||||
error.AppendLine(eventArgs.Data);
|
||||
}
|
||||
};
|
||||
|
||||
bool processStarted = process.Start();
|
||||
|
||||
if (!processStarted)
|
||||
{
|
||||
Tracing.Logger.Log.FailedInstallingAnyBuildClient(m_loggingContext, "Failed to execute bootstrapper script");
|
||||
return false;
|
||||
}
|
||||
|
||||
#if NET5_0_OR_GREATER
|
||||
await process.WaitForExitAsync();
|
||||
#else
|
||||
#pragma warning disable AsyncFixer02
|
||||
await Task.Run(() => process.WaitForExit());
|
||||
#pragma warning restore AsyncFixer02
|
||||
#endif
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
Tracing.Logger.Log.FailedInstallingAnyBuildClient(
|
||||
m_loggingContext,
|
||||
error.ToString() + " (see log for details)");
|
||||
}
|
||||
|
||||
var result = new string[]
|
||||
{
|
||||
$"Exit code: {process.ExitCode}",
|
||||
$"StdOut: {output}",
|
||||
$"StdErr: {error}"
|
||||
};
|
||||
|
||||
Tracing.Logger.Log.FinishedInstallAnyBuild(
|
||||
m_loggingContext,
|
||||
Environment.NewLine + string.Join(Environment.NewLine, result));
|
||||
|
||||
return process.ExitCode == 0;
|
||||
}
|
||||
|
||||
private async Task<string?> DownloadBootstrapperWinAsync(CancellationToken token)
|
||||
{
|
||||
// PowerShell requires the extension to be .ps1.
|
||||
string bootstrapperFile = Path.ChangeExtension(Path.GetTempFileName(), ".ps1");
|
||||
|
||||
try
|
||||
{
|
||||
// Cancellation token on GetStreamAsync is only available starting .NET5.
|
||||
using (var httpClient = new HttpClient())
|
||||
#if NET5_0_OR_GREATER
|
||||
using (var stream = await httpClient.GetStreamAsync($"{m_source}/bootstrapper.ps1", token))
|
||||
#else
|
||||
using (var stream = await httpClient.GetStreamAsync($"{m_source}/bootstrapper.ps1"))
|
||||
#endif
|
||||
using (var bootstrapper = new FileStream(bootstrapperFile, FileMode.Create))
|
||||
{
|
||||
await stream.CopyToAsync(bootstrapper);
|
||||
}
|
||||
|
||||
return bootstrapperFile;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Tracing.Logger.Log.FailedDownloadingAnyBuildClient(m_loggingContext, ex.ToString());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -4,6 +4,7 @@
|
|||
// #define FEATURE_ANYBUILD_PROCESS_REMOTING
|
||||
#if FEATURE_ANYBUILD_PROCESS_REMOTING
|
||||
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using AnyBuild;
|
||||
using BuildXL.Processes.Remoting.AnyBuild;
|
||||
|
|
@ -34,8 +35,15 @@ namespace BuildXL.Processes.Remoting
|
|||
|
||||
private static async Task<IRemoteProcessPipResult> GetCompletionAsync(IRemoteProcess remoteProcess)
|
||||
{
|
||||
IRemoteProcessResult anyBuildResult = await remoteProcess.Completion;
|
||||
return AnyBuildRemoteProcessPipResult.FromAnyBuildResult(anyBuildResult);
|
||||
try
|
||||
{
|
||||
IRemoteProcessResult anyBuildResult = await remoteProcess.Completion;
|
||||
return AnyBuildRemoteProcessPipResult.FromAnyBuildResult(anyBuildResult);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return new ErrorRemoteProcessPipResult(e.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,9 +62,15 @@ namespace BuildXL.Processes.Remoting
|
|||
useLocalEnvironment: false,
|
||||
processInfo.Environments.ToList());
|
||||
|
||||
IRemoteProcess remoteCommand = await factory.CreateAndStartAsync(commandInfo, cancellationToken);
|
||||
|
||||
return new AnyBuildRemoteProcess(remoteCommand);
|
||||
try
|
||||
{
|
||||
IRemoteProcess remoteCommand = await factory.CreateAndStartAsync(commandInfo, cancellationToken);
|
||||
return new AnyBuildRemoteProcess(remoteCommand);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return new ErrorRemoteProcessPip(e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
|
|
@ -119,6 +125,9 @@ namespace BuildXL.Processes.Remoting
|
|||
m_executionContext.CancellationToken,
|
||||
logDirectory: logDir,
|
||||
additionalAnyBuildParameters: extraParams,
|
||||
// TODO: Use available ports instead of the defaults. It may address the issue with /server-.
|
||||
// daemonPort: GetUnusedPort(),
|
||||
// shimPort: GetUnusedPort(),
|
||||
inheritHandlesOnProcessCreation: false);
|
||||
}
|
||||
catch (Exception e)
|
||||
|
|
@ -166,48 +175,28 @@ namespace BuildXL.Processes.Remoting
|
|||
$"--CacheDir {localCacheDir}",
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId))
|
||||
string extraArgs = EngineEnvironmentSettings.AnyBuildExtraArgs;
|
||||
if (!string.IsNullOrEmpty(extraArgs))
|
||||
{
|
||||
args.Add("--ClientApplicationId");
|
||||
args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalAppId);
|
||||
|
||||
if (string.IsNullOrEmpty(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv))
|
||||
{
|
||||
throw new BuildXLException($"Service principal password is required for starting AnyBuild daemon");
|
||||
}
|
||||
|
||||
args.Add("--ClientSecretEnvironmentVariable");
|
||||
args.Add(EngineEnvironmentSettings.AnyBuildServicePrincipalPwdEnv);
|
||||
extraArgs = extraArgs.Replace("~~", " ").Replace("!!", "\"");
|
||||
args.Add(extraArgs);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(m_configuration.Schedule.RemoteExecutionServiceUri))
|
||||
{
|
||||
try
|
||||
{
|
||||
var uri = new Uri(m_configuration.Schedule.RemoteExecutionServiceUri);
|
||||
if (uri.IsLoopback)
|
||||
{
|
||||
args.Add("--Loopback");
|
||||
args.Add("--WaitForAgentForever");
|
||||
}
|
||||
else
|
||||
{
|
||||
args.Add($"--RemoteExecServiceUri {uri}");
|
||||
}
|
||||
}
|
||||
catch (UriFormatException e)
|
||||
{
|
||||
throw new BuildXLException(
|
||||
$"Invalid {nameof(m_configuration.Schedule.RemoteExecutionServiceUri)}: {m_configuration.Schedule.RemoteExecutionServiceUri}",
|
||||
e);
|
||||
}
|
||||
}
|
||||
|
||||
args.Add(EngineEnvironmentSettings.AnyBuildExtraArgs ?? string.Empty);
|
||||
|
||||
return string.Join(" ", args);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IRemoteProcessManagerInstaller? GetInstaller() => new AnyBuildInstaller(m_loggingContext);
|
||||
|
||||
// private static int GetUnusedPort()
|
||||
// {
|
||||
// var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
// listener.Start();
|
||||
// int port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
// listener.Stop();
|
||||
// return port;
|
||||
// }
|
||||
|
||||
private record InitResult(AnyBuildClient AbClient, AnyBuildDaemonManager DaemonManager, IRemoteProcessFactory RemoteProcessFactory);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Threading.Tasks;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace BuildXL.Processes.Remoting
|
||||
{
|
||||
/// <summary>
|
||||
/// Error remote process pip.
|
||||
/// </summary>
|
||||
internal class ErrorRemoteProcessPip : IRemoteProcessPip
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public Task<IRemoteProcessPipResult> Completion { init; get; }
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of <see cref="ErrorRemoteProcessPip"/>.
|
||||
/// </summary>
|
||||
public ErrorRemoteProcessPip(string error) => Completion = Task.FromResult((IRemoteProcessPipResult)new ErrorRemoteProcessPip(error));
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace BuildXL.Processes.Remoting
|
||||
{
|
||||
/// <summary>
|
||||
/// Error result for process remoting.
|
||||
/// </summary>
|
||||
internal class ErrorRemoteProcessPipResult : IRemoteProcessPipResult
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public bool ShouldRunLocally => true;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public int? ExitCode => default;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public string StdOut => string.Empty;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public string StdErr { init; get; }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public RemoteResultDisposition Disposition => RemoteResultDisposition.None;
|
||||
|
||||
/// <summary>
|
||||
/// Creates an instance of <see cref="ErrorRemoteProcessPipResult"/>.
|
||||
/// </summary>
|
||||
public ErrorRemoteProcessPipResult(string error) => StdErr = error;
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,8 @@ using System;
|
|||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace BuildXL.Processes.Remoting
|
||||
{
|
||||
/// <summary>
|
||||
|
|
@ -33,5 +35,11 @@ namespace BuildXL.Processes.Remoting
|
|||
/// Checks if this process manager has been initialized.
|
||||
/// </summary>
|
||||
bool IsInitialized { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Returns an installer for this remote process manager.
|
||||
/// </summary>
|
||||
/// <returns>An instance of <see cref="IRemoteProcessManagerInstaller"/>.</returns>
|
||||
IRemoteProcessManagerInstaller? GetInstaller();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT License.
|
||||
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace BuildXL.Processes.Remoting
|
||||
{
|
||||
/// <summary>
|
||||
/// Interface for installer of remote process manager.
|
||||
/// </summary>
|
||||
public interface IRemoteProcessManagerInstaller
|
||||
{
|
||||
/// <summary>
|
||||
/// Installs remote process manager.
|
||||
/// </summary>
|
||||
/// <param name="token">Cancellation token.</param>
|
||||
/// <param name="forceInstall">Force installation by cleaning existing installation when set to true.</param>
|
||||
/// <returns>True if installation was successful.</returns>
|
||||
Task<bool> InstallAsync(CancellationToken token, bool forceInstall = false);
|
||||
}
|
||||
}
|
||||
|
|
@ -27,6 +27,9 @@ namespace BuildXL.Processes.Remoting
|
|||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IRemoteProcessManagerInstaller? GetInstaller() => null;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public Task InitAsync() => throw new BuildXLException(RemotingNotSupportedMessage);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,11 @@ namespace BuildXL.Processes.Remoting
|
|||
/// </summary>
|
||||
public enum RemoteResultDisposition
|
||||
{
|
||||
/// <summary>
|
||||
/// None.
|
||||
/// </summary>
|
||||
None,
|
||||
|
||||
/// <summary>
|
||||
/// Process completed with cache hit.
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -408,6 +408,42 @@ namespace BuildXL.Processes.Tracing
|
|||
Message = "Exception on getting AnyBuild remote process factory: {exception}")]
|
||||
public abstract void ExceptionOnGetAnyBuildRemoteProcessFactory(LoggingContext context, string exception);
|
||||
|
||||
[GeneratedEvent(
|
||||
(int)LogEventId.InstallAnyBuildClient,
|
||||
EventGenerators = EventGenerators.LocalOnly,
|
||||
EventLevel = Level.Informational,
|
||||
Keywords = (int)Keywords.UserMessage,
|
||||
EventTask = (int)Tasks.PipExecutor,
|
||||
Message = "Installing AnyBuild client from '{source}' (ring: {ring})")]
|
||||
public abstract void InstallAnyBuildClient(LoggingContext context, string source, string ring);
|
||||
|
||||
[GeneratedEvent(
|
||||
(int)LogEventId.FailedDownloadingAnyBuildClient,
|
||||
EventGenerators = EventGenerators.LocalOnly,
|
||||
EventLevel = Level.Warning,
|
||||
Keywords = (int)Keywords.UserMessage,
|
||||
EventTask = (int)Tasks.PipExecutor,
|
||||
Message = "Failed downloading AnyBuild client: {message}")]
|
||||
public abstract void FailedDownloadingAnyBuildClient(LoggingContext context, string message);
|
||||
|
||||
[GeneratedEvent(
|
||||
(int)LogEventId.FailedInstallingAnyBuildClient,
|
||||
EventGenerators = EventGenerators.LocalOnly,
|
||||
EventLevel = Level.Warning,
|
||||
Keywords = (int)Keywords.UserMessage,
|
||||
EventTask = (int)Tasks.PipExecutor,
|
||||
Message = "Failed installing AnyBuild client: {message}")]
|
||||
public abstract void FailedInstallingAnyBuildClient(LoggingContext context, string message);
|
||||
|
||||
[GeneratedEvent(
|
||||
(int)LogEventId.FinishedInstallAnyBuild,
|
||||
EventGenerators = EventGenerators.LocalOnly,
|
||||
EventLevel = Level.Verbose,
|
||||
Keywords = (int)Keywords.UserMessage,
|
||||
EventTask = (int)Tasks.PipExecutor,
|
||||
Message = "Finished installing AnyBuild client: {message}")]
|
||||
public abstract void FinishedInstallAnyBuild(LoggingContext context, string message);
|
||||
|
||||
[GeneratedEvent(
|
||||
(int)LogEventId.LogMacKextFailure,
|
||||
EventGenerators = EventGenerators.LocalOnly,
|
||||
|
|
|
|||
|
|
@ -162,6 +162,10 @@ namespace BuildXL.Processes.Tracing
|
|||
PipProcessFinishedRemoteExecution = 12503,
|
||||
ExceptionOnFindOrStartAnyBuildDaemon = 12504,
|
||||
ExceptionOnGetAnyBuildRemoteProcessFactory = 12505,
|
||||
InstallAnyBuildClient = 12506,
|
||||
FailedDownloadingAnyBuildClient = 12507,
|
||||
FailedInstallingAnyBuildClient = 12508,
|
||||
FinishedInstallAnyBuild = 12509,
|
||||
|
||||
//// Special tool errors
|
||||
PipProcessToolErrorDueToHandleToFileBeingUsed = 14300,
|
||||
|
|
|
|||
|
|
@ -47,6 +47,11 @@ namespace BuildXL.Scheduler.Distribution
|
|||
/// </summary>
|
||||
public int TotalRemoteFallbackRetryLocally => Volatile.Read(ref m_totalRemoteFallbackRetryLocally);
|
||||
|
||||
/// <summary>
|
||||
/// Disable remoting when unexpected scenario (e.g., failed installing AnyBuild) happens.
|
||||
/// </summary>
|
||||
public bool DisableRemoting { get; set; } = false;
|
||||
|
||||
private StringTable StringTable => PipExecutionContext.StringTable;
|
||||
|
||||
private readonly ISandboxConfiguration m_sandboxConfig;
|
||||
|
|
@ -92,7 +97,8 @@ namespace BuildXL.Scheduler.Distribution
|
|||
// When the process requires an admin privilege, then most likely it has to run in a VM hosted
|
||||
// by the local worker. Thus, the parallelism of running such process should be the same as running
|
||||
// the process on the local worker.
|
||||
if (processRunnable.RunLocation == ProcessRunLocation.Local
|
||||
if (DisableRemoting
|
||||
|| processRunnable.RunLocation == ProcessRunLocation.Local
|
||||
|| ProcessRequiresAdminPrivilege(processRunnable.Process)
|
||||
|| (ExistTags(m_processMustRunLocalTags) && HasTag(processRunnable.Process, m_processMustRunLocalTags))
|
||||
|| (ExistTags(m_processCanRunRemoteTags) && !HasTag(processRunnable.Process, m_processCanRunRemoteTags))
|
||||
|
|
|
|||
|
|
@ -6186,6 +6186,17 @@ namespace BuildXL.Scheduler
|
|||
sidebandState: m_sidebandState,
|
||||
serviceManager: m_serviceManager,
|
||||
alienFileEnumerationCache: m_alienFileEnumerationCache);
|
||||
|
||||
if (m_scheduleConfiguration.EnableProcessRemoting)
|
||||
{
|
||||
IRemoteProcessManagerInstaller installer = RemoteProcessManager.GetInstaller();
|
||||
|
||||
if (installer != null && !installer.InstallAsync(m_schedulerCancellationTokenSource.Token).GetAwaiter().GetResult())
|
||||
{
|
||||
// Disable remoting when installation is unsuccessful.
|
||||
((LocalWorkerWithRemoting)LocalWorker).DisableRemoting = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,10 +13,7 @@ namespace ExternalToolTest.BuildXL.Scheduler
|
|||
|
||||
public bool IsInitialized { get; private set; }
|
||||
|
||||
public TestRemoteProcessManager(bool shouldRunLocally)
|
||||
{
|
||||
m_shouldRunLocally = shouldRunLocally;
|
||||
}
|
||||
public TestRemoteProcessManager(bool shouldRunLocally) => m_shouldRunLocally = shouldRunLocally;
|
||||
|
||||
public async Task<IRemoteProcessPip> CreateAndStartAsync(RemoteProcessInfo processInfo, CancellationToken cancellationToken)
|
||||
{
|
||||
|
|
@ -34,5 +31,7 @@ namespace ExternalToolTest.BuildXL.Scheduler
|
|||
IsInitialized = true;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public IRemoteProcessManagerInstaller GetInstaller() => null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,17 +86,29 @@ namespace BuildXL.Utilities.Configuration
|
|||
/// <summary>
|
||||
/// Extra arguments to be passed to AnyBuild daemon for process remoting.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Due to possibly complicated interaction with the batch/powershell scripts that call BuildXL,
|
||||
/// - the space separator can be replaced by double tilde,
|
||||
/// - double quotes can be replaced by double exclamation mark.
|
||||
/// For example, '--NoCheckForUpdates --AnyBuildConfig "C:\path with space\foo.json"' can be written as '--NoCheckForUpdates~~--AnyBuildConfig~~!!C:\path~~with~~space\foo.json!!'.
|
||||
/// </remarks>
|
||||
public static readonly Setting<string> AnyBuildExtraArgs = CreateSetting("BUILDXL_ANYBUILD_EXTRA_ARGS", value => value);
|
||||
|
||||
/// <summary>
|
||||
/// Service principal app id for starting AnyBuild daemon.
|
||||
/// When true, force installation of AnyBuild client by removing existing installation.
|
||||
/// </summary>
|
||||
public static readonly Setting<string> AnyBuildServicePrincipalAppId = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_APP_ID", value => value);
|
||||
/// <remarks>
|
||||
/// This setting is only applicable when BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR is unspecified.
|
||||
/// </remarks>
|
||||
public static readonly Setting<bool> AnyBuildForceInstall = CreateSetting("BUILDXL_ANYBUILD_FORCE_INSTALL", value => value == "1");
|
||||
|
||||
/// <summary>
|
||||
/// Environment variable where the service principal password for starting AnyBuild daemon is stored.
|
||||
/// Source of AnyBuild client installation.
|
||||
/// </summary>
|
||||
public static readonly Setting<string> AnyBuildServicePrincipalPwdEnv = CreateSetting("BUILDXL_ANYBUILD_SERVICE_PRINCIPAL_PWD_ENV", value => value);
|
||||
/// <remarks>
|
||||
/// The source is an URI that can be suffixed with the installation ring specification, e.g., 'https://anybuild.azureedge.net/clientreleases?ring=Dogfood'.
|
||||
/// </remarks>
|
||||
public static readonly Setting<string> AnyBuildClientSource = CreateSetting("BUILDXL_ANYBUILD_SOURCE", value => value);
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whether the application should fail fast on null reference exceptions
|
||||
|
|
|
|||
|
|
@ -501,15 +501,6 @@ namespace BuildXL.Utilities.Configuration
|
|||
/// </summary>
|
||||
double RemotingThresholdMultiplier { get; }
|
||||
|
||||
/// <summary>
|
||||
/// URI of cluster to use for remoting processes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When unspecified, the service URI will be determined by AnyBuild client itself.
|
||||
/// This setting is only applicable when <see cref="EnableProcessRemoting"/> is true.
|
||||
/// </remarks>
|
||||
string RemoteExecutionServiceUri { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether Cpu resource determines the scheduling behavior
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -185,7 +185,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
|
|||
ProcessCanRunRemoteTags = new List<string>(template.ProcessCanRunRemoteTags);
|
||||
ProcessMustRunLocalTags = new List<string>(template.ProcessMustRunLocalTags);
|
||||
RemotingThresholdMultiplier = template.RemotingThresholdMultiplier;
|
||||
RemoteExecutionServiceUri = template.RemoteExecutionServiceUri;
|
||||
|
||||
StopDirtyOnSucceedFastPips = template.StopDirtyOnSucceedFastPips;
|
||||
CpuResourceAware = template.CpuResourceAware;
|
||||
|
|
@ -469,9 +468,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
|
|||
|
||||
private int NumOfRemoteAgentLeasesValue => NumOfRemoteAgentLeases ?? 2 * MaxProcesses;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string RemoteExecutionServiceUri { get; set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public bool CpuResourceAware { get; set; }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,10 @@
|
|||
<#
|
||||
.SYNOPSIS
|
||||
|
||||
Script for BuildXL self-hosting with specified PATs. This script is used to perform BuildXL self-hosting in Azure pipeline.
|
||||
|
||||
#>
|
||||
|
||||
[CmdletBinding(PositionalBinding=$false)]
|
||||
Param(
|
||||
[Parameter(mandatory=$true)]
|
||||
|
|
@ -9,20 +16,20 @@ Param(
|
|||
[Parameter(mandatory=$true)]
|
||||
[String]$MsEngGitPat,
|
||||
|
||||
[ValidateSet("LKG", "Dev", "RunCheckinTests", "RunCheckinTestSamples", "ChangeJournalService")]
|
||||
[string]$Use = "LKG",
|
||||
[ValidateSet("Release", "Debug")]
|
||||
[string]$DeployConfig = "Debug",
|
||||
[ValidateSet("net472", "net5.0", "net6.0", "win-x64", "osx-x64")]
|
||||
[string]$DeployRuntime = "win-x64",
|
||||
[Parameter(Mandatory=$false)]
|
||||
[switch]$DeployDev = $false,
|
||||
[Parameter(Mandatory=$false)]
|
||||
[switch]$UseDev = $false,
|
||||
[Parameter(Mandatory=$false)]
|
||||
[ValidateSet("Dev", "RunCheckinTests", "RunCheckinTestSamples", "ChangeJournalService")]
|
||||
[string]$Deploy,
|
||||
[switch]$Minimal = $false,
|
||||
[Parameter(Mandatory=$false)]
|
||||
[switch]$Release = $false,
|
||||
|
||||
[Parameter(Mandatory=$false)]
|
||||
[switch]$EnableProcessRemoting = $false,
|
||||
[Parameter(Mandatory=$false)]
|
||||
[string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
|
||||
[Parameter(Mandatory=$false)]
|
||||
[string]$AnyBuildClientDir,
|
||||
|
||||
[Parameter(mandatory=$false, ValueFromRemainingArguments=$true)]
|
||||
|
|
@ -42,21 +49,15 @@ Param(
|
|||
]
|
||||
}", "Process")
|
||||
|
||||
$BxlCmdArgs = @()
|
||||
$BxlCmdArgs = @(
|
||||
"-Use", $Use,
|
||||
"-DeployConfig", $DeployConfig,
|
||||
"-DeployRuntime", $DeployRuntime
|
||||
)
|
||||
|
||||
if ($UseDev)
|
||||
if (-not [string]::IsNullOrEmpty($Deploy))
|
||||
{
|
||||
$BxlCmdArgs += @("-Use", "Dev")
|
||||
}
|
||||
|
||||
if ($DeployDev)
|
||||
{
|
||||
$BxlCmdArgs += "-DeployDev"
|
||||
}
|
||||
|
||||
if ($Release)
|
||||
{
|
||||
$BxlCmdArgs += @("-DeployConfig", "Release")
|
||||
$BxlCmdArgs += @("-Deploy", $Deploy)
|
||||
}
|
||||
|
||||
if ($Minimal)
|
||||
|
|
@ -67,7 +68,6 @@ if ($Minimal)
|
|||
if ($EnableProcessRemoting)
|
||||
{
|
||||
$BxlCmdArgs += "-EnableProcessRemoting"
|
||||
$BxlCmdArgs += @("-RemoteServiceUri", $RemoteServiceUri)
|
||||
}
|
||||
|
||||
if (-not [string]::IsNullOrEmpty($AnyBuildClientDir))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
Function Install-AnyBuild
|
||||
{
|
||||
param(
|
||||
[string]$AnyBuildSource,
|
||||
[string]$Ring,
|
||||
[bool]$Clean = $false
|
||||
);
|
||||
|
||||
$AbDir = "$env:LOCALAPPDATA\Microsoft\AnyBuild"
|
||||
$AbCmd = "$AbDir\AnyBuild.cmd"
|
||||
|
||||
if ($Clean)
|
||||
{
|
||||
Remove-Item -Force -Recurse "$AbDir" -ea SilentlyContinue
|
||||
}
|
||||
|
||||
if (Test-Path $AbCmd -PathType Leaf)
|
||||
{
|
||||
Write-Host "AnyBuild client is already installed"
|
||||
return
|
||||
}
|
||||
|
||||
Write-Host "Install AnyBuild from $AnyBuildSource ($Ring)"
|
||||
|
||||
# Make lowercase as AzStorage cannot handle uppercase in URL.
|
||||
$source = $AnyBuildSource.ToLowerInvariant()
|
||||
$bootstrapperArgs = @("$source", "$Ring")
|
||||
|
||||
Write-Host "Bootstrapper args: '$bootstrapperArgs'";
|
||||
|
||||
while ($true)
|
||||
{
|
||||
$script = ((curl.exe -s -S --retry 10 --retry-connrefused "$source/bootstrapper.ps1") | Out-String)
|
||||
|
||||
if ($LASTEXITCODE -eq 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
# We sometimes get a 404 error when downloading the bootstrapper
|
||||
# while the bootstrapper script is updating. Needs an eventual fix instead of this workaround.
|
||||
Write-Host "ERROR: Failed downloading the bootstrapper script. Trying again."
|
||||
}
|
||||
|
||||
Invoke-Command -ScriptBlock ([scriptblock]::Create($script)) -ArgumentList $bootstrapperArgs
|
||||
}
|
||||
|
|
@ -151,9 +151,6 @@ param(
|
|||
[Parameter(Mandatory=$false)]
|
||||
[switch]$EnableProcessRemoting = $false,
|
||||
|
||||
[Parameter(Mandatory=$false)]
|
||||
[string]$RemoteServiceUri = "https://westus2.anybuild-test.microsoft.com/clusters/07F427C5-7979-415C-B6D9-01BAD5118191",
|
||||
|
||||
[Parameter(Mandatory=$false)]
|
||||
[string]$AnyBuildClientDir,
|
||||
|
||||
|
|
@ -219,6 +216,8 @@ $isMicrosoftInternal = [Environment]::GetEnvironmentVariable("[Sdk.BuildXL]micro
|
|||
# - /ado option is present, so AzDevOps scenarios are kept unchanged.
|
||||
# - this is not considered an internal build
|
||||
# We might decide to relax this once shared compilation gets enough mileage.
|
||||
# TODO: Enable shared compilation for -EnableProcessRemoting.
|
||||
# Currently some builds failed to write outputs. Need more investigation.
|
||||
if ($UseManagedSharedCompilation -and
|
||||
(($DominoArguments -like '*/ado*') -or (-not $isMicrosoftInternal) -or $EnableProcessRemoting)) {
|
||||
$UseManagedSharedCompilation = $false
|
||||
|
|
@ -324,10 +323,17 @@ if (($DominoArguments -match "logsDirectory:.*").Length -eq 0 -and ($DominoArgum
|
|||
if ($EnableProcessRemoting) {
|
||||
# Unit tests are not supported for remoting because change journal is not enabled on agents
|
||||
# and all volumes in agents have the same serial.
|
||||
$AdditionalBuildXLArguments += @("/server-", "/enableLazyOutputs-", "/exp:lazysodeletion-", "/enableProcessRemoting+", "/processCanRunRemoteTags:compile", "/processMustRunLocalTags:telemetry:xUnit;telemetry:xUnitUntracked;telemetry:QTest", "/remoteExecutionServiceUri:$RemoteServiceUri");
|
||||
$AdditionalBuildXLArguments += @(
|
||||
# TODO: Remove /server-
|
||||
# Currently needed because of the following exception:
|
||||
# Exception:System.Net.Internals.SocketExceptionFactory+ExtendedSocketException (10061): No connection could be made because the target machine actively refused it. [::ffff:127.0.0.1]:21337
|
||||
"/server-",
|
||||
"/enableProcessRemoting+",
|
||||
"/processCanRunRemoteTags:compile",
|
||||
"/processMustRunLocalTags:telemetry:xUnit;telemetry:xUnitUntracked;telemetry:QTest");
|
||||
|
||||
if (-not [string]::IsNullOrEmpty($AnyBuildClientDir)) {
|
||||
$AdditionalBuildXLArguments += " /p:BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR=$AnyBuildClientDir"
|
||||
$AdditionalBuildXLArguments += " /p:BUILDXL_ANYBUILD_CLIENT_INSTALL_DIR=`"$AnyBuildClientDir`""
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -727,11 +733,9 @@ if ($env:BUILDXL_ADDITIONAL_DEFAULTS)
|
|||
[string[]]$DominoArguments = @($DominoArguments |% { $_.Replace("#singlequote#", "'").Replace("#openparens#", "(").Replace("#closeparens#", ")"); })
|
||||
[string[]]$DominoArguments = $AdditionalBuildXLArguments + $DominoArguments;
|
||||
|
||||
if ($NoSubst)
|
||||
{
|
||||
if ($NoSubst) {
|
||||
$bxlExitCode = Run-ProcessWithoutNormalizedPath $useDeployment.domino $DominoArguments;
|
||||
} else
|
||||
{
|
||||
} else {
|
||||
$bxlExitCode = Run-ProcessWithNormalizedPath $useDeployment.dominoRunner $useDeployment.domino $DominoArguments;
|
||||
}
|
||||
$bxlSuccess = ($bxlExitCode -eq 0);
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче