microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 634680: Process-remoting-aware scheduler 

Add a worker that is aware of process remoting capability.

All of these changes are needed to speed up building tenant Word with process remoting via AnyBuild

Related work items: #1906258
This commit is contained in:
Iman Narasamdya 2022-01-06 01:15:49 +00:00
Родитель 221fc90670
Коммит 46030cb8d4
44 изменённых файлов: 1964 добавлений и 555 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

26
Public/Src/App/Bxl/Args.cs
Просмотреть файл

@ -238,6 +238,9 @@ namespace BuildXL
OptionHandlerFactory.CreateBoolOption(
"analyzeDependencyViolations",
opt => { /* DEPRECATED -- DO NOTHING */ }),
OptionHandlerFactory.CreateOption(
"augmentingPathSetCommonalityFactor",
opt => cacheConfiguration.AugmentWeakFingerprintRequiredPathCommonalityFactor = CommandLineUtilities.ParseDoubleOption(opt, 0, 1)),
OptionHandlerFactory.CreateBoolOption(
"breakOnUnexpectedFileAccess",
opt => sandboxConfiguration.BreakOnUnexpectedFileAccess = opt),
@ -352,7 +355,7 @@ namespace BuildXL
opt => loggingConfiguration.DumpFailedPips = opt),
OptionHandlerFactory.CreateOption(
"dumpFailedPipsLogLimit",
opt => loggingConfiguration.DumpFailedPipsLogLimit = CommandLineUtilities.ParseInt32Option(opt, 0, Int32.MaxValue)),
opt => loggingConfiguration.DumpFailedPipsLogLimit = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
OptionHandlerFactory.CreateBoolOption(
"dumpFailedPipsWithDynamicData",
opt => loggingConfiguration.DumpFailedPipsWithDynamicData = opt),
@ -438,6 +441,9 @@ namespace BuildXL
OptionHandlerFactory.CreateBoolOption(
"enableLessAggresiveMemoryProjection",
sign => schedulingConfiguration.EnableLessAggresiveMemoryProjection = sign),
OptionHandlerFactory.CreateBoolOption(
"enableProcessRemoting",
sign => schedulingConfiguration.EnableProcessRemoting = sign),
OptionHandlerFactory.CreateBoolOption(
"enableSetupCostWhenChoosingWorker",
sign => schedulingConfiguration.EnableSetupCostWhenChoosingWorker = sign),
@ -792,6 +798,9 @@ namespace BuildXL
OptionHandlerFactory.CreateOption(
"noWarn",
opt => ParseInt32ListOption(opt, loggingConfiguration.NoWarnings)),
OptionHandlerFactory.CreateOption(
"numRemoteAgentLeases",
opt => schedulingConfiguration.NumOfRemoteAgentLeases = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
OptionHandlerFactory.CreateOption(
"numRetryFailedPipsOnAnotherWorker",
opt => distributionConfiguration.NumRetryFailedPipsOnAnotherWorker = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
@ -830,9 +839,6 @@ namespace BuildXL
OptionHandlerFactory.CreateOption(
"pathSetThreshold",
opt => cacheConfiguration.AugmentWeakFingerprintPathSetThreshold = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
OptionHandlerFactory.CreateOption(
"augmentingPathSetCommonalityFactor",
opt => cacheConfiguration.AugmentWeakFingerprintRequiredPathCommonalityFactor = CommandLineUtilities.ParseDoubleOption(opt, 0, 1)),
OptionHandlerFactory.CreateOption(
"pathSetAugmentationMonitoring",
opt => cacheConfiguration.MonitorAugmentedPathSets = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
@ -865,6 +871,12 @@ namespace BuildXL
OptionHandlerFactory.CreateOption(
"printFile2FileDependencies",
opt => frontEndConfiguration.FileToFileReportDestination = CommandLineUtilities.ParsePathOption(opt, pathTable)),
OptionHandlerFactory.CreateOption(
"processCanRunRemoteTags",
opt => schedulingConfiguration.ProcessCanRunRemoteTags.AddRange(CommandLineUtilities.ParseRepeatingOption(opt, ";", s => s.Trim()))),
OptionHandlerFactory.CreateOption(
"processMustRunLocalTags",
opt => schedulingConfiguration.ProcessMustRunLocalTags.AddRange(CommandLineUtilities.ParseRepeatingOption(opt, ";", s => s.Trim()))),
OptionHandlerFactory.CreateOption(
"processRetries",
opt => schedulingConfiguration.ProcessRetries = CommandLineUtilities.ParseInt32Option(opt, 0, int.MaxValue)),
@ -890,15 +902,15 @@ namespace BuildXL
OptionHandlerFactory.CreateOption(
"relatedActivityId",
opt => loggingConfiguration.RelatedActivityId = CommandLineUtilities.ParseStringOption(opt)),
OptionHandlerFactory.CreateBoolOption(
"remoteAllProcesses",
sign => sandboxConfiguration.RemoteAllProcesses = sign),
OptionHandlerFactory.CreateBoolOptionWithValue(
"remoteTelemetry",
(opt, sign) =>
loggingConfiguration.RemoteTelemetry =
CommandLineUtilities.ParseBoolEnumOption(opt, sign, RemoteTelemetry.EnabledAndNotify, RemoteTelemetry.Disabled),
isEnabled: () => loggingConfiguration.RemoteTelemetry.HasValue && loggingConfiguration.RemoteTelemetry.Value != RemoteTelemetry.Disabled),
OptionHandlerFactory.CreateOption(
"remotingThresholdMultiplier",
opt => schedulingConfiguration.RemotingThresholdMultiplier = CommandLineUtilities.ParseDoubleOption(opt, 0, double.MaxValue)),
OptionHandlerFactory.CreateBoolOption(
"replaceExistingFileOnMaterialization",
sign => cacheConfiguration.ReplaceExistingFileOnMaterialization = sign),

38
Public/Src/App/Bxl/HelpText.cs
Просмотреть файл

@ -896,6 +896,40 @@ namespace BuildXL
HelpLevel.Verbose);
#endregion
hw.WriteBanner(
Strings.HelpText_DisplayHelp_ProcessRemotingBanner,
HelpLevel.Verbose);
#region Process Remoting
hw.WriteOption(
"/enableProcessRemoting[+|-]",
Strings.HelpText_DisplayHelp_EnableProcessRemoting,
HelpLevel.Verbose);
hw.WriteOption(
"/processCanRunRemoteTags:<semi-colon separated tags>",
Strings.HelpText_DisplayHelp_ProcessCanRunRemoteTags,
HelpLevel.Verbose);
hw.WriteOption(
"/processMustRunLocalTags:<semi-colon separated tags>",
Strings.HelpText_DisplayHelp_ProcessMustRunLocalTags,
HelpLevel.Verbose);
hw.WriteOption(
"/remotingThresholdMultiplier:<double>",
Strings.HelpText_DisplayHelp_RemotingThresholdMultiplier,
HelpLevel.Verbose);
hw.WriteOption(
"/numRemoteAgentLeases:<int>",
Strings.HelpText_DisplayHelp_NumRemoteAgentLeases,
HelpLevel.Verbose);
#endregion
hw.WriteBanner(Strings.HelpText_DisplayHelp_DiagBanner);
#region Diagnostics
@ -1182,12 +1216,12 @@ namespace BuildXL
hw.WriteOption(
"/relatedActivityId:<guid>",
Strings.HelpText_DiplayHelp_RelatedActivityId,
Strings.HelpText_DisplayHelp_RelatedActivityId,
HelpLevel.Verbose);
hw.WriteOption(
"/vs[+|-]",
Strings.HelpText_DiplayHelp_VS);
Strings.HelpText_DisplayHelp_VS);
hw.WriteOption(
"/solutionName:<string>",

24
Public/Src/App/Bxl/Strings.resx
Просмотреть файл

@ -616,7 +616,7 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_EngineCacheDirectory" xml:space="preserve">
<value>Allows overriding where engine state will be cached. If unset, it will be stored in a subdirectory of the artifact cache.</value>
</data>
<data name="HelpText_DiplayHelp_RelatedActivityId" xml:space="preserve">
<data name="HelpText_DisplayHelp_RelatedActivityId" xml:space="preserve">
<value>An external related ETW activity identifier. The top level {ShortProductName} activity will be logged as a child of this one.</value>
</data>
<data name="HelpText_DisplayHelp_Filter_Input" xml:space="preserve">
@ -742,7 +742,7 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_EnforceFileAccesses" xml:space="preserve">
<value>Whether {ShortProductName} is to monitor file accesses of individual tools at all. Disabling monitoring results in an unsafe configuration (for diagnostic purposes only). Defaults to on.</value>
</data>
<data name="HelpText_DiplayHelp_VS" xml:space="preserve">
<data name="HelpText_DisplayHelp_VS" xml:space="preserve">
<value>Generates a VS solution file and MSBuild files for C# and C++ projects. Defaults to off.</value>
</data>
<data name="HelpText_DisplayHelp_SolutionName" xml:space="preserve">
@ -1081,10 +1081,28 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
<data name="HelpText_DisplayHelp_DumpFailedPipsWithDynamicData" xml:space="preserve">
<value>Enable this option to dump observed file accesses and processes with the dump pip lite analyzer (requires /logObservedFileAccesses+ and/or /logProcesses+ to be set as well).</value>
</data>
<data name="HelpText_DisplayHelp_EnforceFullReparsePointsUnderPath" xml:space="preserve">
<data name="HelpText_DisplayHelp_EnforceFullReparsePointsUnderPath" xml:space="preserve">
<value>Enforce that files accessed which begin with the given path will enforce reparse points underneath said path. All transitive reparse points encountered after enforcing and resolving the first one are also enforced, regardless of path.</value>
</data>
<data name="HelpText_DisplayHelp_BuildManifestVerifyFileContentOnHashComputation" xml:space="preserve">
<value>When enabled, ensures that file's content matches the hash provided by the engine before proceeding to compute a build manifest hash for that file.</value>
</data>
<data name="HelpText_DisplayHelp_EnableProcessRemoting" xml:space="preserve">
<value>Enable process remoting via AnyBuild. Defaults to off.</value>
</data>
<data name="HelpText_DisplayHelp_ProcessCanRunRemoteTags" xml:space="preserve">
<value>Tags for processes that can run remotely when process remoting is enabled. When unspecified, every process can be remoted, unless it has a tag specified in /processMustRunLocalTags. </value>
</data>
<data name="HelpText_DisplayHelp_ProcessMustRunLocalTags" xml:space="preserve">
<value>Tags for processes that must run locally when process remoting is enabled. When unspecified, it is assumed to be empty.</value>
</data>
<data name="HelpText_DisplayHelp_ProcessRemotingBanner" xml:space="preserve">
<value>PROCESS REMOTING</value>
</data>
<data name="HelpText_DisplayHelp_NumRemoteAgentLeases" xml:space="preserve">
<value>Static number of remote agent leases. Only applicable when /enableProcessRemoting is set to true. Defaults to 2 * /maxProc.</value>
</data>
<data name="HelpText_DisplayHelp_RemotingThresholdMultiplier" xml:space="preserve">
<value>Multiplier for threshold before starting to remote process pips when /enableProcessRemoting is set to true. The threshold is obtained by multiplying /maxProc with this multiplier. Defaults to 1.5.</value>
</data>
</root>

15
Public/Src/App/ConsoleLogger/ConsoleEventListener.cs
Просмотреть файл

@ -259,6 +259,7 @@ namespace BuildXL
var copyFileNotDone = (long)payload[21];
var writeFileDone = (long)payload[22];
var writeFileNotDone = (long)payload[23];
var remoteProcs = (long)payload[24];
long done = pipsSucceeded + pipsFailed + pipsSkipped;
long total = done + pipsRunning + pipsWaiting + pipsReady;
@ -288,7 +289,14 @@ namespace BuildXL
sb.Append(@" {{6,{0}}} skipped,");
}
sb.Append(@" {{8,{0}}} executing, {{2,{0}}} waiting]");
if (remoteProcs > 0)
{
sb.Append(@" {{8,{0}}} executing ({{14}} remote), {{2,{0}}} waiting]");
}
else
{
sb.Append(@" {{8,{0}}} executing, {{2,{0}}} waiting]");
}
if (pipsWaitingOnSemaphore > 0)
{
@ -309,7 +317,7 @@ namespace BuildXL
sb.Length = 0;
var format = FinalizeFormatStringLayout(sb, statusLine, 0);
sb.AppendFormat(
CultureInfo.InvariantCulture,
format,
@ -326,7 +334,8 @@ namespace BuildXL
done,
total,
filePipsDone,
filePipsTotal);
filePipsTotal,
remoteProcs);
if (pipsWaitingOnResources > 0)
{

3
Public/Src/App/UnitTests/Bxl/DevOpsListenerTests.cs
Просмотреть файл

@ -244,7 +244,8 @@ namespace Test.BuildXL
copyFileDone:100,
copyFileNotDone: 100,
writeFileDone: 10,
writeFileNotDone:10);
writeFileNotDone:10,
procsRemoted: 0);
console.ValidateCall(MessageLevel.Info, $"##vso[task.setprogress value={currentProgress};]Pip Execution phase");
}
}

15
Public/Src/Engine/Processes/ExternalSandboxedProcess.cs
Просмотреть файл

@ -118,6 +118,21 @@ namespace BuildXL.Processes
}
}
/// <summary>
/// Some sandboxed processes may want to clean up the whole working directory to prevent it from becoming large.
/// </summary>
/// <remarks>
/// Note that this method does not clean up the working directory of the underlying process. It cleans up the
/// working directory of the external sandboxed process executor that is defined in the constructor of <see cref="ExternalSandboxedProcess"/>.
/// The reason for this is to keep the entries of the root of working directories small. Particularly when the process needs
/// to execute remotely, the directory entries of the root need to be sent to the remote agent. The more entries, the bigger the data
/// needs to be sent remotely.
/// </remarks>
protected void CleanUpWorkingDirectory()
{
FileUtilities.DeleteDirectoryContents(WorkingDirectory, deleteRootDirectory: true);
}
/// <summary>
/// Throws an instance of <see cref="BuildXLException"/>.
/// </summary>

6
Public/Src/Engine/Processes/ExternalToolSandboxedProcess.cs
Просмотреть файл

@ -19,8 +19,8 @@ namespace BuildXL.Processes
{
private readonly ExternalToolSandboxedProcessExecutor m_tool;
private readonly StringBuilder m_output = new StringBuilder();
private readonly StringBuilder m_error = new StringBuilder();
private readonly StringBuilder m_output = new ();
private readonly StringBuilder m_error = new ();
private AsyncProcessExecutor m_processExecutor;
@ -104,7 +104,7 @@ namespace BuildXL.Processes
{
FileName = m_tool.ExecutablePath,
Arguments = m_tool.CreateArguments(SandboxedProcessInfoFile, SandboxedProcessResultsFile),
WorkingDirectory = SandboxedProcessInfo.WorkingDirectory,
WorkingDirectory = WorkingDirectory,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,

5
Public/Src/Engine/Processes/FileAccessManifest.cs
Просмотреть файл

@ -531,6 +531,11 @@ namespace BuildXL.Processes
/// When no manifest path is found, the returned manifest path is <see cref="AbsolutePath.Invalid"/>.
/// The node policy is always set (and will contain the policy of the root node if no explicit
/// manifest path is found)
///
/// TODO: BUG 1904974
/// When the manifest is obtained from a deserialization, it will have a new empty path table.
/// This method only works if the given path is obtained from the same path table that the original
/// file access manifest holds before being serialized.
/// </remarks>
public bool TryFindManifestPathFor(AbsolutePath path, out AbsolutePath manifestPath, out FileAccessPolicy nodePolicy)
{

31
Public/Src/Engine/Processes/ProcessRunLocation.cs Normal file
Просмотреть файл

@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Processes
{
/// <summary>
/// Location where the process will run.
/// </summary>
public enum ProcessRunLocation
{
/// <summary>
/// Use the default configuration; most likely local (see <see cref="Local"/>).
/// </summary>
Default,
/// <summary>
/// Process is forced to run locally on the same machine.
/// </summary>
/// <remarks>
/// The process can be run as an immediate child process of BuildXL, i.e., it is executed directly by <see cref="SandboxedProcessPipExecutor"/>.
/// Or, the process can be run using the external sandboxed process executor, but still on the same machine.
/// Or, the process can be run in the VM hosted by the machine.
/// </remarks>
Local,
/// <summary>
/// Process is expected to run on a remote agent via AnyBuild.
/// </summary>
Remote,
}
}

31
Public/Src/Engine/Processes/Remoting/AnyBuild/CommandExecutionDisposition.cs Normal file
Просмотреть файл

@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// Various execution sub-results provided from AnyBuild process execution.
/// </summary>
public enum CommandExecutionDisposition
{
/// <summary>
/// Default zero value.
/// </summary>
Unknown,
/// <summary>
/// Process completed with cache hit.
/// </summary>
CacheHit,
/// <summary>
/// The process was remoted to an AnyBuild cluster.
/// </summary>
Remoted,
/// <summary>
/// The process was run locally by AnyBuild.exe and no fallback execution is needed.
/// </summary>
RanLocally,
}
}

64
Public/Src/Engine/Processes/Remoting/Protocol.cs → Public/Src/Engine/Processes/Remoting/AnyBuild/Protocol.cs
Просмотреть файл

@ -1,12 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.IO;
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// CODESYNC: AnyBuild src/Client/Shim.Shared/Protocol.cs
/// Constants, helper methods, and descriptions for the protocol format between the Shim and AnyBuild.
/// </summary>
public static class Protocol
/// <remarks>
/// The protocol between AnyBuild and the shim each use a length-prefixed string format. This is designed
/// not for minimizing size but to minimize encoding time - we need to get message information between
/// processes quickly, and memcpy is faster than UTF-8 conversion.
///
/// The first 4 bytes in each packet are a direct copy of the int32 count of bytes in the remainder of
/// the message. The message payload is UTF-16 characters as a string. There is no null termination of
/// strings except where a message requires it.
///
/// Example when sending the string "Smessage" (a stdout message from AnyBuild to Shim) across the wire is below.
/// The string is 8 characters long, so it requires 16 bytes in the payload, so the prefix integer is 16.
///
/// ----Message size--- ---'S'--- ---'m'--- ---'e'--- ---'s'--- ---'s'--- ---'a'--- ---'g'--- ---'e'---
/// 0x10 0x00 0x00 0x00 0x53 0x00 0x6D 0x00 0x65 0x00 0x73 0x00 0x73 0x00 0x61 0x00 0x67 0x00 0x65 0x00
///
/// .
/// </remarks>
internal static class Protocol
{
/// <summary>
/// AnyBuild -> Shim message that tells the Shim to run the process locally.
@ -25,10 +44,49 @@ namespace BuildXL.Processes.Remoting
/// <summary>
/// AnyBuild -> Shim string message prefix that indicates the remote process has completed.
/// It includes a return code, e.g. "C0" for a zero return code.
/// It includes the following semicolon-delimited fields:
/// - A process return/exit code
/// - A disposition, one of 'C' for a cache hit, 'R' for remoting, 'L' for local execution.
/// </summary>
public const char ProcessCompleteMessagePrefix = 'C';
public static string CreateProcessCompleteMessageSuffix(int exitCode, CommandExecutionDisposition disposition)
{
return $"{exitCode};{Disposition.ToProtocolDisposition(disposition)}";
}
/// <summary>
/// Execution dispositions. See <see cref="ProcessCompleteMessagePrefix"/>.
/// </summary>
public static class Disposition
{
public const char CacheHit = 'C';
public const char Remoted = 'R';
public const char RanLocally = 'L';
public static char ToProtocolDisposition(CommandExecutionDisposition disposition)
{
return disposition switch
{
CommandExecutionDisposition.CacheHit => CacheHit,
CommandExecutionDisposition.Remoted => Remoted,
CommandExecutionDisposition.RanLocally => RanLocally,
_ => throw new InvalidDataException($"Unexpected disposition {disposition}"),
};
}
public static CommandExecutionDisposition ToCommandExecutionDisposition(char protocolDisposition)
{
return protocolDisposition switch
{
CacheHit => CommandExecutionDisposition.CacheHit,
Remoted => CommandExecutionDisposition.Remoted,
RanLocally => CommandExecutionDisposition.RanLocally,
_ => throw new InvalidDataException($"Unexpected disposition {protocolDisposition}"),
};
}
}
/// <summary>
/// Shim -> AnyBuild string message prefix that runs a process. The following string is a null character
/// delimited set of fields in the following order:

345
Public/Src/Engine/Processes/Remoting/AnyBuild/RemoteCommand.cs Normal file
Просмотреть файл

@ -0,0 +1,345 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Utilities;
#nullable enable
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// Virtual factory interface for creating an instance of <see cref="IRemoteProcess"/>.
/// </summary>
public interface IRemoteProcessFactory
{
/// <summary>
/// Creates an instance of <see cref="IRemoteProcess"/>, starting the remote execution without waiting for process completion.
/// </summary>
/// <param name="remoteProcessInfo">Process info.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>An instance of <see cref="IRemoteProcess"/> that can be used to await process completion or cancel.</returns>
Task<IRemoteProcess> CreateAndStartAsync(RemoteCommandExecutionInfo remoteProcessInfo, CancellationToken cancellationToken);
}
/// <summary>
/// The results of a completed <see cref="IRemoteProcess"/>.
/// </summary>
public interface IRemoteProcessResult
{
/// <summary>
/// Whether the process should be run locally because of some failure remoting.
/// </summary>
public bool ShouldRunLocally { get; }
/// <summary>
/// Gets the process exit code, or null if the process was not remotable.
/// </summary>
int? ExitCode { get; }
/// <summary>
/// Gets the stdout contents of the process, or null if the process was not remotable.
/// </summary>
string? StdOut { get; }
/// <summary>
/// Gets the stderr contents of the process, or null if the process was not remotable.
/// </summary>
string? StdErr { get; }
/// <summary>
/// Gets whether the process was a cache hit or was remoted.
/// </summary>
CommandExecutionDisposition Disposition { get; }
}
/// <summary>
/// Concrete implementation of <see cref="IRemoteProcessResult"/>.
/// </summary>
internal sealed class RemoteProcessResult : IRemoteProcessResult
{
/// <inheritdoc/>
public bool ShouldRunLocally { get; private set; }
/// <inheritdoc/>
public int? ExitCode { get; private set; }
/// <inheritdoc/>
public string? StdOut { get; private set; }
/// <inheritdoc/>
public string? StdErr { get; private set; }
/// <inheritdoc/>
public CommandExecutionDisposition Disposition { get; private set; }
internal static RemoteProcessResult CreateForLocalRun() => new() { ShouldRunLocally = true };
internal static RemoteProcessResult CreateFromCompletedProcess(int exitCode, string stdOut, string stdErr, CommandExecutionDisposition disposition) =>
new() { ExitCode = exitCode, StdOut = stdOut, StdErr = stdErr, Disposition = disposition };
}
/// <summary>
/// Runs a remote process via AnyBuild. Process file outputs are placed onto the local disk.
/// </summary>
public interface IRemoteProcess : IDisposable
{
/// <summary>
/// Allows awaiting remote processing completion.
/// </summary>
/// <exception cref="TaskCanceledException">
/// The caller-provided cancellation token was signaled or the object was disposed.
/// </exception>
Task<IRemoteProcessResult> Completion { get; }
}
/// <summary>
/// Concrete virtual factory implementation for <see cref="IRemoteProcessFactory"/>.
/// </summary>
public sealed class RemoteProcessFactory : IRemoteProcessFactory
{
/// <summary>
/// Singleton instance for the factory.
/// </summary>
public static RemoteProcessFactory Instance { get; } = new();
/// <summary>
/// Creates and starts an instance of <see cref="IRemoteProcess"/>.
/// </summary>
public Task<IRemoteProcess> CreateAndStartAsync(RemoteCommandExecutionInfo remoteProcessInfo, CancellationToken cancellationToken)
{
return CreateAndStartAsync(21337, remoteProcessInfo, cancellationToken);
}
/// <summary>
/// Creates and starts an instance of <see cref="IRemoteProcess"/>.
/// </summary>
[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1822:MarkMembersAsStatic", Justification = "Virtual factory pattern")]
public async Task<IRemoteProcess> CreateAndStartAsync(int shimPort, RemoteCommandExecutionInfo remoteProcessInfo, CancellationToken cancellationToken, IShimClient? shimClient = null)
{
var cmd = new RemoteCommand(shimPort, remoteProcessInfo, cancellationToken, shimClient);
await cmd.StartAsync();
return cmd;
}
}
/// <summary>
/// Runs a remote command over the Shim execution protocol, storing the resulting stdout, stderr, and exit code.
/// Command file outputs are placed into the local disk by the AnyBuild service process.
/// </summary>
/// <remarks>
/// CODESYNC:
/// - (AnyBuild) src/Client/AnyBuild/ShimServer.cs
/// - (BuildXL) src/Engine/Processes/Remoting/RemotingSandboxedProcess.cs
/// </remarks>
public sealed class RemoteCommand : IRemoteProcess
{
private readonly IShimClient m_client;
private readonly RemoteCommandExecutionInfo m_commandInfo;
private Task<IRemoteProcessResult>? m_processingTask;
private readonly StringBuilder m_stdout = new (128);
private readonly StringBuilder m_stderr = new ();
private readonly CancellationTokenSource m_cancelProcessCts = new ();
private readonly CancellationTokenSource m_combinedCts;
/// <summary>
/// Creates an instance of <see cref="RemoteCommand"/>.
/// </summary>
/// <param name="port">The TCP port of the localhost AnyBuild service process.</param>
/// <param name="command">Parameters for process execution.</param>
/// <param name="cancellationToken">A cancellation token for the command execution.</param>
/// <param name="shimClient">A Shim client, for unit testing. By default a new <see cref="ShimClient"/> is used. This client will be disposed on dispose of this object.</param>
internal RemoteCommand(
int port,
RemoteCommandExecutionInfo command,
CancellationToken cancellationToken,
IShimClient? shimClient = null)
{
m_combinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_cancelProcessCts.Token);
m_client = shimClient ?? new ShimClient(port, m_combinedCts.Token);
m_commandInfo = command;
}
/// <inheritdoc/>
public void Dispose()
{
CancelAsync().GetAwaiter().GetResult();
m_client.Dispose();
m_combinedCts.Dispose();
m_cancelProcessCts.Dispose();
}
/// <summary>
/// Allows for awaiting remote process completion.
/// </summary>
public Task<IRemoteProcessResult> Completion => m_processingTask!;
/// <summary>
/// Starts the remote process without waiting for process completion.
/// Should only be called from <see cref="RemoteProcessFactory"/>.
/// </summary>
internal async Task StartAsync()
{
long startTicks = DateTime.UtcNow.Ticks;
long connectTicks = DateTime.UtcNow.Ticks;
if (!m_client.IsConnected)
{
await m_client.ConnectAsync();
}
connectTicks = DateTime.UtcNow.Ticks - connectTicks;
string requestString = CreateRequestString(startTicks, connectTicks);
await m_client.WriteAsync(Protocol.RunProcessMessagePrefix, requestString);
m_processingTask = Task.Run(() => ProcessResponseAsync(), m_combinedCts.Token);
}
/// <summary>
/// Cancels remote process execution. This also causes <see cref="Completion"/> to complete.
/// </summary>
public async Task CancelAsync()
{
if (m_processingTask != null)
{
if (!m_processingTask.IsCompleted)
{
m_cancelProcessCts.Cancel();
}
try
{
await m_processingTask;
}
#pragma warning disable ERP022
catch
{
// Eat shutdown exceptions.
}
#pragma warning restore ERP022
m_processingTask = null;
}
}
private async Task<IRemoteProcessResult> ProcessResponseAsync()
{
bool done = false;
while (!done && !m_combinedCts.IsCancellationRequested)
{
string response = await m_client.ReceiveStringAsync();
if (string.IsNullOrEmpty(response))
{
throw new InvalidDataException("Received null or empty response");
}
switch (response[0])
{
case Protocol.RunBuildLocallyMessage:
return RemoteProcessResult.CreateForLocalRun();
case Protocol.ProcessCompleteMessagePrefix:
int semicolonIndex = response.IndexOf(';');
string exitCodeStr = response.Substring(1, semicolonIndex - 1);
int exitCode = int.Parse(exitCodeStr, CultureInfo.InvariantCulture);
CommandExecutionDisposition disposition = Protocol.Disposition.ToCommandExecutionDisposition(response[semicolonIndex + 1]);
return RemoteProcessResult.CreateFromCompletedProcess(exitCode, m_stdout.ToString(), m_stderr.ToString(), disposition);
case Protocol.StdoutMessagePrefix:
{
// Trim \0 at end
string s = StripPrefixAndNullChar(response);
m_stdout.AppendLine(s);
break;
}
case Protocol.StderrMessagePrefix:
{
string s = StripPrefixAndNullChar(response);
m_stderr.AppendLine(s);
break;
}
default:
throw new InvalidDataException($"Unknown message protocol '{response[0]}'");
}
}
// Should not be reachable but need safe return.
return RemoteProcessResult.CreateForLocalRun();
}
private static string StripPrefixAndNullChar(string protocolString)
{
return protocolString.Substring(1, protocolString.Length - 2);
}
private string CreateRequestString(long startTicks, long connectTicks)
{
var builder = new StringBuilder(128);
// We're not replacing an existing process with a shim so use a default value.
const string ShimmedProcessId = "0";
builder.AppendShimString(ShimmedProcessId);
// Parent process ID (this process).
builder.AppendShimString(Process.GetCurrentProcess().Id.ToString(CultureInfo.InvariantCulture));
AppendCommandAndArgs(builder)
.AppendShimString(m_commandInfo.WorkingDirectory)
.AppendShimString(startTicks.ToString(CultureInfo.InvariantCulture));
AppendEnv(builder)
.AppendShimString(connectTicks.ToString(CultureInfo.InvariantCulture))
.AppendShimString(DateTime.UtcNow.Ticks.ToString(CultureInfo.InvariantCulture));
return builder.ToString();
}
private StringBuilder AppendCommandAndArgs(StringBuilder builder)
{
if (OperatingSystemHelper.IsLinuxOS)
{
builder
.AppendShimString(m_commandInfo.Command)
.AppendShimString(m_commandInfo.Args);
}
else
{
builder.Append(m_commandInfo.Command).Append(' ').AppendShimString(m_commandInfo.Args);
}
return builder;
}
private StringBuilder AppendEnv(StringBuilder builder)
{
foreach (KeyValuePair<string, string> kvp in m_commandInfo.Env)
{
builder.Append(kvp.Key).Append('=').AppendShimString(kvp.Value);
}
// Double null at end of list.
builder.Append('\0');
return builder;
}
}
internal static class ShimStringBuilderExtensions
{
public static StringBuilder AppendShimString(this StringBuilder sb, string s)
{
sb.Append(s).Append('\0');
return sb;
}
}
}

73
Public/Src/Engine/Processes/Remoting/AnyBuild/RemoteCommandExecutionInfo.cs Normal file
Просмотреть файл

@ -0,0 +1,73 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections;
using System.Collections.Generic;
using BuildXL.Utilities;
#nullable enable
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// Parameters for process execution.
/// </summary>
/// <remarks>
/// CODESYNC: (AnyBuild) src/Client/ClientLibNetStd/Shim/RemoteCommandExecutionInfo.cs
/// </remarks>
public sealed class RemoteCommandExecutionInfo
{
/// <nodoc/>
public RemoteCommandExecutionInfo(
string command,
string? args,
string cwd,
bool useLocalEnvironment,
IEnumerable<KeyValuePair<string, string>>? environmentVariablesToAdd)
{
Command = command;
Args = args ?? string.Empty;
WorkingDirectory = cwd;
var env = new Dictionary<string, string>(OperatingSystemHelper.EnvVarComparer);
if (useLocalEnvironment)
{
#pragma warning disable CS8605
#pragma warning disable CS8601
#pragma warning disable CS8602
#pragma warning disable CS8604
foreach (DictionaryEntry kvp in Environment.GetEnvironmentVariables())
{
env[kvp.Key.ToString()] = kvp.Value.ToString();
}
#pragma warning restore CS8604
#pragma warning restore CS8602
#pragma warning restore CS8601
#pragma warning restore CS8605
}
if (environmentVariablesToAdd != null)
{
foreach (KeyValuePair<string, string> kvp in environmentVariablesToAdd)
{
env[kvp.Key] = kvp.Value;
}
}
Env = env;
}
/// <nodoc/>
public string Command { get; }
/// <nodoc/>
public string Args { get; }
/// <nodoc/>
public string WorkingDirectory { get; }
/// <nodoc/>
public IReadOnlyDictionary<string, string> Env { get; }
}
}

205
Public/Src/Engine/Processes/Remoting/AnyBuild/ShimClient.cs Normal file
Просмотреть файл

@ -0,0 +1,205 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Utilities;
#nullable enable
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// Low-level protocol interface for communicating to an AnyBuild shim server running on a loopback socket.
/// </summary>
/// <remarks>
/// CODESYNC:
/// - (AnyBuild) src/Client/ClientLibNetStd/Shim/ShimClient.css
/// </remarks>
public interface IShimClient : IDisposable
{
/// <summary>
/// Indicates whether a connection exists to the AnyBuild shim server.
/// </summary>
bool IsConnected { get; }
/// <summary>
/// Connects to the AnyBuild service process.
/// </summary>
Task ConnectAsync();
/// <summary>
/// Awaits a string message from the service process.
/// </summary>
Task<string> ReceiveStringAsync();
/// <summary>
/// Sends a string message to the service process, with message type prefix.
/// </summary>
Task WriteAsync(char messageTypePrefix, string message);
}
/// <summary>
/// Low-level class wrapping TCP client for sending message to AnyBuild shim server.
/// <see cref="RemoteCommand"/> is a higher-level class that implements remote execution
/// and caching of individual processes/commands.
/// </summary>
/// <remarks>
/// CODESYNC:
/// - (AnyBuild) src/Client/ClientLibNetStd/Shim/ShimClient.css
/// </remarks>
internal sealed class ShimClient : IShimClient, IDisposable
{
private static readonly Encoding s_messageEncoding = OperatingSystemHelper.IsLinuxOS
? Encoding.UTF8
: Encoding.Unicode;
private static readonly int s_messageEncodingBytesPerChar = s_messageEncoding.GetByteCount(new[] { 'c' });
internal const int DefaultReceiveBufferSizeChars = 52 * 1024;
private static readonly int s_defaultReceiveBufferSize = DefaultReceiveBufferSizeChars * s_messageEncodingBytesPerChar;
private readonly int m_port;
private readonly CancellationToken m_cancellationToken;
private NetworkStream? m_networkStream;
private readonly TcpClient m_tcpClient = new() { NoDelay = true };
private byte[] m_writeBuffer = new byte[s_defaultReceiveBufferSize];
private byte[] m_readBuffer = new byte[s_defaultReceiveBufferSize];
/// <summary>
/// Creates an instance of <see cref="ShimClient"/>.
/// </summary>
/// <param name="port">The TCP port of the localhost AnyBuild service process.</param>
/// <param name="cancellationToken">A cancellation token for this client's actions.</param>
public ShimClient(int port, CancellationToken cancellationToken)
{
m_port = port;
m_cancellationToken = cancellationToken;
}
/// <nodoc/>
public bool IsConnected => m_tcpClient.Connected;
/// <nodoc/>
public async Task ConnectAsync()
{
await m_tcpClient.ConnectAsync(IPAddress.Loopback, m_port);
if (!m_tcpClient.Connected)
{
throw new InvalidOperationException("BUGBUG: Client is not connected");
}
m_networkStream = m_tcpClient.GetStream();
}
/// <nodoc/>
public async Task<string> ReceiveStringAsync()
{
CheckConnected();
int offset = 0;
int payloadSize = -1;
// Get the first 4 bytes containing the payload size.
while (payloadSize < 0)
{
int bytesRead = await m_networkStream!.ReadAsync(m_readBuffer, offset, 4 - offset, m_cancellationToken);
if (bytesRead == 0)
{
// Socket closed. Unexpected if we have not received data yet.
if (offset > 0)
{
throw new InvalidOperationException("Unexpected close during message receive");
}
}
offset += bytesRead;
if (offset == 4)
{
payloadSize = BitConverter.ToInt32(m_readBuffer, 0);
}
}
if (payloadSize > m_readBuffer.Length)
{
m_readBuffer = new byte[payloadSize];
}
offset = 0;
while (offset < payloadSize)
{
int bytesRead = await m_networkStream!.ReadAsync(m_readBuffer, offset, payloadSize - offset, m_cancellationToken);
if (bytesRead == 0)
{
// Socket closed.
throw new OperationCanceledException("Unexpected close during message receive");
}
offset += bytesRead;
}
return s_messageEncoding.GetString(m_readBuffer, 0, payloadSize);
}
/// <summary>
/// Writes a prefix character and a string. Note that the socket server typically needs a terminating null character
/// on the string, and that this is the caller's responsibility.
/// </summary>
public Task WriteAsync(char messageTypePrefix, string message)
{
CheckConnected();
int writeBufferStart = 4; // Length field
int totalSize = writeBufferStart + (1 /*messageType*/ + message.Length) * s_messageEncodingBytesPerChar;
if (m_writeBuffer.Length < totalSize)
{
m_writeBuffer = new byte[totalSize];
}
unsafe
{
fixed (byte* pBuf = &m_writeBuffer[0])
{
*(int*)pBuf = totalSize - writeBufferStart;
}
}
writeBufferStart += s_messageEncoding.GetBytes(new[] { messageTypePrefix }, 0, 1, m_writeBuffer, writeBufferStart);
writeBufferStart += s_messageEncoding.GetBytes(message, 0, message.Length, m_writeBuffer, writeBufferStart);
if (writeBufferStart != totalSize)
{
throw new InvalidOperationException($"BUGBUG: Packet length mismatch, estimated {totalSize}, from encoding {writeBufferStart}");
}
return m_networkStream!.WriteAsync(m_writeBuffer, 0, totalSize, m_cancellationToken);
}
/// <inheritdoc/>
public void Dispose()
{
try
{
m_tcpClient.Dispose();
}
#pragma warning disable ERP022
catch
{
// May have self-disposed on close.
}
#pragma warning restore ERP022
m_networkStream?.Dispose();
}
private void CheckConnected()
{
if (!IsConnected)
{
throw new InvalidOperationException(nameof(IsConnected) + " is false");
}
}
}
}

204
Public/Src/Engine/Processes/Remoting/RemoteSandboxedProcess.cs
Просмотреть файл

@ -5,12 +5,9 @@ using System;
using System.Diagnostics.ContractsLight;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Interop;
using BuildXL.Utilities;
namespace BuildXL.Processes.Remoting
{
@ -19,50 +16,40 @@ namespace BuildXL.Processes.Remoting
/// </summary>
/// <remarks>
/// This sandboxed process adds process remoting capability for BuildXL via AnyBuild's shim server.
/// When set with /remoteAllProcesses, for each process, BuildXL will create a TCP client that
/// sends a build request to the AnyBuild's shim server. The remoting mechanism leverages the SandboxedProcessExecutor
/// used for executing processes in VM. That is, instead of executing the process directly, AnyBuild's shim server
/// will be instructed to execute SandboxedProcessExecutor given the process' SandboxedProcessInfo.
/// The remoting mechanism leverages the SandboxedProcessExecutor used for executing processes in VM. That is, instead of executing
/// the process directly, AnyBuild will be instructed to execute SandboxedProcessExecutor given the process' SandboxedProcessInfo.
/// </remarks>
public class RemoteSandboxedProcess : ExternalSandboxedProcess
{
/// <summary>
/// CODESYNC: AnyBuild src/Client/Shim.Shared/ShimConstants.cs
/// Remote process factory creator.
/// </summary>
private const string DefaultPortEnv = "__ANYBUILD_PORT";
public static Lazy<IRemoteProcessFactory> RemoteProcessFactory;
private IRemoteProcess m_remoteProcess;
private readonly CancellationTokenSource m_killProcessCts = new ();
private readonly CancellationTokenSource m_combinedCts;
private readonly CancellationToken m_cancellationToken;
private readonly ExternalToolSandboxedProcessExecutor m_tool;
private readonly StringBuilder m_output = new StringBuilder();
private readonly StringBuilder m_error = new StringBuilder();
private RemotingClient m_remotingClient;
private readonly CancellationTokenSource m_cancellationTokenSource = new CancellationTokenSource();
private readonly CancellationToken m_cancellationToken;
private bool? m_shouldRunLocally = default;
private int? m_exitCode = default;
private Task m_processingTask;
private bool IsProcessingCompleted => m_processingTask != null && m_processingTask.IsCompleted;
private bool IsCompletedSuccessfully => m_remoteProcess != null && m_remoteProcess.Completion.IsCompleted && m_remoteProcess.Completion.Status == TaskStatus.RanToCompletion;
/// <summary>
/// Checks if process should be run locally.
/// </summary>
public bool? ShouldRunLocally => IsProcessingCompleted ? m_shouldRunLocally : default;
public bool? ShouldRunLocally => IsCompletedSuccessfully && m_remoteProcess.Completion.Result.ShouldRunLocally;
/// <inheritdoc />
public override int ProcessId => 0;
/// <inheritdoc />
public override string StdOut => IsProcessingCompleted ? m_output.ToString() : string.Empty;
public override string StdOut => IsCompletedSuccessfully ? (m_remoteProcess.Completion.Result.StdOut ?? string.Empty) : string.Empty;
/// <inheritdoc />
public override string StdErr => IsProcessingCompleted ? m_error.ToString() : string.Empty;
public override string StdErr => IsCompletedSuccessfully ? (m_remoteProcess.Completion.Result.StdErr ?? string.Empty) : string.Empty;
/// <inheritdoc />
public override int? ExitCode => IsProcessingCompleted ? m_exitCode : default;
public override int? ExitCode => IsCompletedSuccessfully ? m_remoteProcess.Completion.Result.ExitCode : default;
/// <summary>
/// Creates an instance of <see cref="RemoteSandboxedProcess"/>.
@ -77,129 +64,46 @@ namespace BuildXL.Processes.Remoting
Contract.Requires(tool != null);
m_tool = tool;
m_cancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_cancellationTokenSource.Token).Token;
m_cancellationToken = cancellationToken;
m_combinedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, m_killProcessCts.Token);
if (RemoteProcessFactory == null)
{
RemoteProcessFactory = new Lazy<IRemoteProcessFactory>(() =>
{
// TODO:
// var clientDll = Assembly.Load(... path to client dll...);
// s_remoteProcessFactory = Activator.CreateInstance(clientDll.GetType("AnyBuild.BuildXL.Client.BuildXLRemoteProcessFactory"));
return new RemoteProcessFactory();
});
}
}
/// <inheritdoc />
public override void Start()
{
long startTicks = DateTime.UtcNow.Ticks;
base.Start();
SerializeSandboxedProcessInputFile(SandboxedProcessInfoFile, SandboxedProcessInfo.Serialize);
var remoteProcessInfo = new RemoteCommandExecutionInfo(
m_tool.ExecutablePath,
m_tool.CreateArguments(SandboxedProcessInfoFile, SandboxedProcessResultsFile),
SandboxedProcessInfo.WorkingDirectory,
useLocalEnvironment: false,
SandboxedProcessInfo.EnvironmentVariables.ToDictionary().ToList());
m_remotingClient = new RemotingClient(GetEndPoint(), m_cancellationToken);
long connectTicks = DateTime.UtcNow.Ticks;
m_remotingClient.ConnectAsync().GetAwaiter().GetResult();
connectTicks = DateTime.UtcNow.Ticks - connectTicks;
string requestString = CreateRequestString(startTicks, connectTicks);
m_remotingClient.WriteAsync(Protocol.RunProcessMessagePrefix, requestString).GetAwaiter().GetResult();
m_processingTask = Task.Run(() => ProcessResponseAsync(), m_cancellationToken);
}
private async Task ProcessResponseAsync()
{
bool done = false;
while (!done && !m_cancellationToken.IsCancellationRequested)
{
string response = await m_remotingClient.ReceiveStringAsync();
if (string.IsNullOrEmpty(response))
{
throw new BuildXLException("Receiving null or empty response");
}
switch (response[0])
{
case Protocol.RunBuildLocallyMessage:
m_shouldRunLocally = true;
done = true;
break;
case Protocol.ProcessCompleteMessagePrefix:
m_exitCode = int.Parse(response.Substring(1));
done = true;
break;
case Protocol.StdoutMessagePrefix:
m_output.AppendLine(response.Substring(1).Trim('\0'));
break;
case Protocol.StderrMessagePrefix:
m_error.AppendLine(response.Substring(1).Trim('\0'));
break;
default:
throw new BuildXLException($"Unknown message protocol '{response[0]}'");
}
}
}
private string CreateRequestString(long startTicks, long connectTicks)
{
var builder = new StringBuilder();
AppendWithCommand(builder)
.Append(CreateString(SandboxedProcessInfo.WorkingDirectory))
.Append(CreateTicks(startTicks))
.Append(CreateEnvString())
.Append(CreateTicks(connectTicks))
.Append(CreateTicks(DateTime.UtcNow.Ticks));
return builder.ToString();
}
private StringBuilder AppendWithCommand(StringBuilder builder)
{
var executable = m_tool.ExecutablePath;
var args = m_tool.CreateArguments(SandboxedProcessInfoFile, SandboxedProcessResultsFile);
builder.Append(CreateString(ProcessId.ToString()));
builder.Append(CreateString(0.ToString()));
if (OperatingSystemHelper.IsLinuxOS)
{
builder
.Append(CreateString(executable))
.Append(CreateString(args));
}
else
{
builder.Append(CreateString(executable + " " + args));
}
return builder;
}
private string CreateEnvString()
{
return CreateString(string.Join(
string.Empty,
SandboxedProcessInfo.EnvironmentVariables.ToDictionary().Select(kvp => CreateString($"{kvp.Key}={kvp.Value}"))));
}
private static string CreateString(string s) => s + "\0";
private static string CreateTicks(long ticks) => CreateString(ticks.ToString());
private static IPEndPoint GetEndPoint()
{
string portString = Environment.GetEnvironmentVariable(DefaultPortEnv);
if (string.IsNullOrEmpty(portString))
{
throw new BuildXLException("Unable to find server port for remoting process");
}
return new IPEndPoint(IPAddress.Loopback, int.Parse(portString));
m_remoteProcess = RemoteProcessFactory.Value.CreateAndStartAsync(remoteProcessInfo, m_combinedCts.Token).GetAwaiter().GetResult();
}
/// <inheritdoc />
public override void Dispose()
{
if (m_remotingClient != null)
{
m_remotingClient.Dispose();
}
m_remoteProcess?.Dispose();
m_combinedCts.Dispose();
m_killProcessCts.Dispose();
m_cancellationTokenSource.Dispose();
CleanUpWorkingDirectory();
}
/// <inheritdoc />
@ -208,10 +112,12 @@ namespace BuildXL.Processes.Remoting
/// <inheritdoc />
public override async Task<SandboxedProcessResult> GetResultAsync()
{
await m_processingTask;
Contract.Assert(m_remoteProcess != null);
if (m_cancellationToken.IsCancellationRequested
|| !IsProcessingCompleted
await m_remoteProcess.Completion;
if (IsCancellationRequested
|| !IsCompletedSuccessfully
|| !ExitCode.HasValue
|| ExitCode.Value != 0)
{
@ -222,12 +128,18 @@ namespace BuildXL.Processes.Remoting
}
/// <inheritdoc />
public override Task KillAsync()
public override async Task KillAsync()
{
m_cancellationTokenSource.Cancel();
return Task.CompletedTask;
m_killProcessCts.Cancel();
if (m_remoteProcess != null)
{
await m_remoteProcess.Completion;
}
}
private bool IsCancellationRequested => m_cancellationToken.IsCancellationRequested || m_killProcessCts.IsCancellationRequested;
/// <inheritdoc />
public override EmptyWorkingSetResult TryEmptyWorkingSet(bool isSuspend) => EmptyWorkingSetResult.None;
@ -236,18 +148,16 @@ namespace BuildXL.Processes.Remoting
private SandboxedProcessResult CreateResultForFailure()
{
string output = m_output.ToString();
string error = m_error.ToString();
string hint = Path.GetFileNameWithoutExtension(m_tool.ExecutablePath);
return CreateResultForFailure(
exitCode: m_cancellationToken.IsCancellationRequested
exitCode: IsCancellationRequested
? ExitCodes.Killed
: (IsProcessingCompleted ? (ExitCode ?? -1) : -1),
killed: m_cancellationToken.IsCancellationRequested,
: (IsCompletedSuccessfully ? (ExitCode ?? -1) : -1),
killed: IsCancellationRequested,
timedOut: false,
output: output,
error: error,
output: StdOut,
error: StdErr,
hint: hint);
}
}

165
Public/Src/Engine/Processes/Remoting/RemoteSandboxedProcessData.cs
Просмотреть файл

@ -1,9 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.IO;
using System.Linq;
using BuildXL.Utilities;
using BuildXL.Utilities.Collections;
namespace BuildXL.Processes.Remoting
{
@ -27,25 +31,180 @@ namespace BuildXL.Processes.Remoting
/// </remarks>
public readonly IReadOnlyList<string> TempDirectories;
/// <summary>
/// Untracked scopes.
/// </summary>
/// <remarks>
/// This is experimental. This data is needed to filter inputs/outputs observed by AnyBuild, so that, particularly for outputs,
/// AnyBuild does not try to send them back to the client. We introduced this data to handle OACR pips, where they have shared accesses
/// on the same untracked output. When running locally, those pips succeed, but when they run remotely in isolation, they fail because
/// the output has incorrect content.
///
/// TODO: Since we will run OACR pips without sharing, this data may no longer be needed.
/// </remarks>
public readonly IReadOnlySet<string> UntrackedScopes;
/// <summary>
/// Untracked paths.
/// </summary>
/// <remarks>
/// See remarks on <see cref="UntrackedScopes"/>.
/// </remarks>
public readonly IReadOnlySet<string> UntrackedPaths;
/// <summary>
/// Creates an instance of <see cref="RemoteSandboxedProcessData"/>.
/// </summary>
/// <param name="tempDirectories">List of temp directories.</param>
public RemoteSandboxedProcessData(IReadOnlyList<string> tempDirectories)
/// <param name="untrackedScopes">Untracked scopes.</param>
/// <param name="untrackedPaths">Untracked paths.</param>
public RemoteSandboxedProcessData(
IReadOnlyList<string> tempDirectories,
IReadOnlySet<string> untrackedScopes,
IReadOnlySet<string> untrackedPaths)
{
Contract.Requires(tempDirectories != null);
Contract.Requires(untrackedScopes != null);
Contract.Requires(untrackedPaths != null);
TempDirectories = tempDirectories;
UntrackedScopes = untrackedScopes;
UntrackedPaths = untrackedPaths;
}
/// <summary>
/// Serializes this instance using an instance of <see cref="BuildXLWriter"/>.
/// </summary>
public void Serialize(BuildXLWriter writer) => writer.WriteReadOnlyList(TempDirectories, (w, s) => w.Write(s));
public void Serialize(BuildXLWriter writer)
{
writer.WriteReadOnlyList(TempDirectories, (w, s) => w.Write(s));
writer.Write(UntrackedScopes, (w, s) => w.Write(s));
writer.Write(UntrackedPaths, (w, s) => w.Write(s));
}
/// <summary>
/// Deserializes an instance of <see cref="RemoteSandboxedProcessData"/> using an instance of <see cref="BuildXLReader"/>.
/// </summary>
public static RemoteSandboxedProcessData Deserialize(BuildXLReader reader) =>
new RemoteSandboxedProcessData(reader.ReadReadOnlyList(r => r.ReadString()));
new (reader.ReadReadOnlyList(r => r.ReadString()),
reader.ReadReadOnlySet(r => r.ReadString()),
reader.ReadReadOnlySet(r => r.ReadString()));
/// <summary>
/// Checks if a file access path is untracked.
/// </summary>
public bool IsUntracked(string fileAccessPath)
{
string normalizedPath = NormalizedInputPathOrNull(fileAccessPath);
if (normalizedPath == null)
{
return false;
}
if (UntrackedPaths.Contains(normalizedPath))
{
return true;
}
foreach (string scope in UntrackedScopes)
{
if (IsWithin(scope, normalizedPath))
{
return true;
}
}
return false;
}
private static string NormalizedInputPathOrNull(string path)
{
if (path == null || path.Length == 0 || path.StartsWith(@"\\.\", StringComparison.Ordinal))
{
return null;
}
int skipStart = 0;
int skipEnd = 0;
if (path.StartsWith(@"\??\", StringComparison.Ordinal)
|| path.StartsWith(@"\\?\", StringComparison.Ordinal))
{
skipStart = 4;
}
if (path[path.Length - 1] == Path.DirectorySeparatorChar)
{
skipEnd = 1;
}
else if (path.EndsWith(@"\..", StringComparison.Ordinal))
{
skipEnd = 3;
}
else if (path.EndsWith(@"\.", StringComparison.Ordinal))
{
skipEnd = 2;
}
int len = path.Length - skipEnd - skipStart;
if (len < 4)
{
// Just a drive letter and colon, or "c:\" which is similar in result.
return null;
}
return path.Substring(skipStart, len);
}
private static bool IsWithin(string parentDir, string path)
{
if (!path.StartsWith(parentDir, OperatingSystemHelper.PathComparison))
{
return false;
}
if (path.Length > parentDir.Length && path[parentDir.Length] != Path.DirectorySeparatorChar)
{
return false;
}
return true;
}
/// <summary>
/// Builder for <see cref="RemoteSandboxedProcessData"/>.
/// </summary>
public class Builder
{
private readonly ReadOnlyHashSet<string> m_untrackedScopes = new (OperatingSystemHelper.PathComparer);
private readonly ReadOnlyHashSet<string> m_untrackedPaths = new (OperatingSystemHelper.PathComparer);
private readonly HashSet<string> m_tempDirectories = new (OperatingSystemHelper.PathComparer);
private readonly PathTable m_pathTable;
/// <summary>
/// Constructor.
/// </summary>
public Builder(PathTable pathTable) => m_pathTable = pathTable;
/// <summary>
/// Adds an untracked scope.
/// </summary>
public void AddUntrackedScope(AbsolutePath path) => m_untrackedScopes.Add(path.ToString(m_pathTable));
/// <summary>
/// Adds an untracked path.
/// </summary>
public void AddUntrackedPath(AbsolutePath path) => m_untrackedPaths.Add(path.ToString(m_pathTable));
/// <summary>
/// Adds a temp directory.
/// </summary>
public void AddTempDirectory(AbsolutePath path) => m_tempDirectories.Add(path.ToString(m_pathTable));
/// <summary>
/// Builds an instance of <see cref="RemoteSandboxedProcessData"/>.
/// </summary>
public RemoteSandboxedProcessData Build() => new (m_tempDirectories.ToList(), m_untrackedScopes, m_untrackedPaths);
}
}
}

155
Public/Src/Engine/Processes/Remoting/RemotingClient.cs
Просмотреть файл

@ -1,155 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Utilities;
using Contract = System.Diagnostics.ContractsLight.Contract;
namespace BuildXL.Processes.Remoting
{
/// <summary>
/// Class wrapping TCP client for sending message to AnyBuild shim server.
/// </summary>
/// <remarks>
/// CODESYNC:
/// - (AnyBuild) src/Client/AnyBuild/SocketClientContext.cs
/// - (AnyBuild) src/Client/AnyBuild/AsyncSocketServer.cs
/// </remarks>
internal class RemotingClient : IDisposable
{
private static readonly Encoding s_messageEncoding = OperatingSystemHelper.IsLinuxOS
? Encoding.UTF8
: Encoding.Unicode;
private static readonly int s_messageEncodingBytesPerChar = s_messageEncoding.GetByteCount(new[] { 'c' });
private static readonly int s_defaultReceiveBufferSize = 52 * 1024 * s_messageEncodingBytesPerChar;
private readonly TcpClient m_tcpClient;
private readonly IPEndPoint m_endPoint;
private readonly CancellationToken m_cancellationToken;
private NetworkStream m_networkStream;
private byte[] m_writeBuffer;
private byte[] m_readBuffer;
public bool IsConnected => m_tcpClient.Connected;
/// <summary>
/// Creates an instance of <see cref="RemotingClient"/>.
/// </summary>
public RemotingClient(IPEndPoint endPoint, CancellationToken cancellationToken)
{
m_endPoint = endPoint;
m_tcpClient = new TcpClient { NoDelay = true };
m_cancellationToken = cancellationToken;
m_writeBuffer = new byte[s_defaultReceiveBufferSize];
m_readBuffer = new byte[s_defaultReceiveBufferSize];
}
internal async Task ConnectAsync()
{
try
{
await m_tcpClient.ConnectAsync(m_endPoint.Address, m_endPoint.Port);
Contract.Assert(m_tcpClient.Connected);
m_networkStream = m_tcpClient.GetStream();
}
catch (Exception e)
{
throw new BuildXLException($"{nameof(RemotingClient)}: Failed to connect", e);
}
}
internal async Task<string> ReceiveStringAsync()
{
Contract.Requires(IsConnected);
int offset = 0;
int payloadSize = -1;
// Get the first 4 bytes containing the payload size.
while (payloadSize < 0)
{
int bytesRead = await m_networkStream.ReadAsync(m_readBuffer, offset, 4 - offset, m_cancellationToken);
if (bytesRead == 0)
{
// Socket closed. Unexpected if we have not received data yet.
if (offset > 0)
{
throw new BuildXLException($"{nameof(RemotingClient)}: Unexpected close during message receive");
}
}
offset += bytesRead;
if (offset == 4)
{
payloadSize = BitConverter.ToInt32(m_readBuffer, 0);
}
}
if (payloadSize > m_readBuffer.Length)
{
m_readBuffer = new byte[payloadSize];
}
offset = 0;
while (offset < payloadSize)
{
int bytesRead = await m_networkStream.ReadAsync(m_readBuffer, offset, payloadSize - offset, m_cancellationToken);
if (bytesRead == 0)
{
// Socket closed.
throw new BuildXLException($"{nameof(RemotingClient)}: Unexpected close during message receive");
}
offset += bytesRead;
}
return s_messageEncoding.GetString(m_readBuffer, 0, payloadSize);
}
public Task WriteAsync(char messageTypePrefix, string message)
{
Contract.Requires(IsConnected);
int writeBufferStart = 4;
int totalSize = writeBufferStart + (1 + message.Length) * s_messageEncodingBytesPerChar;
if (m_writeBuffer.Length < totalSize)
{
m_writeBuffer = new byte[totalSize];
}
unsafe
{
fixed (byte* pBuf = &m_writeBuffer[0])
{
*(int*)pBuf = totalSize - writeBufferStart;
}
}
writeBufferStart += s_messageEncoding.GetBytes(new[] { messageTypePrefix }, 0, 1, m_writeBuffer, writeBufferStart);
writeBufferStart += s_messageEncoding.GetBytes(message, 0, message.Length, m_writeBuffer, writeBufferStart);
return m_networkStream.WriteAsync(m_writeBuffer, 0, totalSize, m_cancellationToken);
}
/// <inheritdoc />
public void Dispose()
{
try
{
m_tcpClient.Dispose();
}
#pragma warning disable ERP022 // Unobserved exception in generic exception handler
catch
{
// May have self-disposed on close.
}
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
}
}
}

186
Public/Src/Engine/Processes/RetryInfo.cs
Просмотреть файл

@ -2,89 +2,72 @@
// Licensed under the MIT License.
using System.Diagnostics.ContractsLight;
using BuildXL.Utilities;
using System.IO;
namespace BuildXL.Processes
{
/// <summary>
/// Some Failing Processes can be retried.
/// This class defines the details required for the retry.
/// Information for process retrial, including the reason and the location (same worker vs. different worker) for retry.
/// </summary>
public class RetryInfo
{
/// <summary>
/// Reason for retry
/// Reason for retry.
/// </summary>
public RetryReason RetryReason { get; }
/// <summary>
/// Location for retry.
/// Caller can decide to retry at a different location
/// Mode for retry.
/// </summary>
public RetryLocation RetryLocation { get; }
public RetryMode RetryMode { get; }
/// <nodoc/>
private RetryInfo(RetryReason retryReason, RetryLocation retryLocation)
/// <summary>
/// Constructor.
/// </summary>
private RetryInfo(RetryReason retryReason, RetryMode retryMode)
{
RetryReason = retryReason;
RetryLocation = retryLocation;
RetryMode = retryMode;
}
/// <summary>
/// Returns a RetryInfo object to be retried on the Same Worker
/// </summary>
private static RetryInfo RetryOnSameWorker(RetryReason retryReason)
{
return new RetryInfo(retryReason, RetryLocation.SameWorker);
}
private static RetryInfo RetryInline(RetryReason retryReason) => new (retryReason, RetryMode.Inline);
private static RetryInfo RetryByReschedule(RetryReason retryReason) => new (retryReason, RetryMode.Reschedule);
private static RetryInfo RetryInlineFirstThenByReschedule(RetryReason retryReason) => new (retryReason, RetryMode.Both);
/// <summary>
/// Returns a RetryInfo object to be retried on a Different Worker
/// Checks if a process can be retried inline.
/// </summary>
private static RetryInfo RetryOnDifferentWorker(RetryReason retryReason)
{
return new RetryInfo(retryReason, RetryLocation.DifferentWorker);
}
public bool CanBeRetriedInline() => RetryMode.CanBeRetriedInline();
/// <summary>
/// Returns a RetryInfo object to be retried on the Same Worker first, and retried on another worker of it fails again
/// Checks if a process can be retried by requeuing it to the scheduler.
/// </summary>
private static RetryInfo RetryOnSameAndDifferentWorkers(RetryReason retryReason)
{
return new RetryInfo(retryReason, RetryLocation.Both);
}
private bool RetryAbleOnSameWorker() => RetryLocation == RetryLocation.SameWorker || RetryLocation == RetryLocation.Both;
private bool RetryAbleOnDifferentWorker() => RetryLocation == RetryLocation.DifferentWorker || RetryLocation == RetryLocation.Both;
public bool CanBeRetriedByReschedule() => RetryMode.CanBeRetriedByReschedule();
/// <summary>
/// Returns true if retry location is <see cref="RetryLocation.SameWorker"/> or <see cref="RetryLocation.Both"/> after a null check
/// Serializes this instance through an instance of <see cref="BinaryWriter"/>.
/// </summary>
public static bool RetryAbleOnSameWorker(RetryInfo retryInfo) => retryInfo?.RetryAbleOnSameWorker() ?? false;
/// <summary>
/// Returns true if retry location is <see cref="RetryLocation.DifferentWorker"/> or <see cref="RetryLocation.Both"/> after a null check
/// </summary>
public static bool RetryAbleOnDifferentWorker(RetryInfo retryInfo) => retryInfo?.RetryAbleOnDifferentWorker() ?? false;
/// <nodoc/>
public void Serialize(BuildXLWriter writer)
public void Serialize(BinaryWriter writer)
{
writer.Write((int)RetryReason);
writer.Write((int)RetryLocation);
writer.Write((int)RetryMode);
}
/// <nodoc/>
public static RetryInfo Deserialize(BuildXLReader reader)
/// <summary>
/// Deserializes an instance of <see cref="RetryInfo"/> from an instance of <see cref="BinaryReader"/>.
/// </summary>
public static RetryInfo Deserialize(BinaryReader reader)
{
var retryReason = (RetryReason)reader.ReadInt32();
var retryLocation = (RetryLocation)reader.ReadInt32();
var retryLocation = (RetryMode)reader.ReadInt32();
return new RetryInfo(retryReason, retryLocation);
}
/// <summary>
/// Returns a RetryInfo object with the default location for the given RetryReason.
/// Gets the default retry information (e.g., retry location) for a given <see cref="RetryReason"/>.
/// </summary>
public static RetryInfo GetDefault(RetryReason reason)
{
@ -94,56 +77,76 @@ namespace BuildXL.Processes
case RetryReason.ProcessStartFailure:
case RetryReason.TempDirectoryCleanupFailure:
case RetryReason.StoppedWorker:
return RetryOnDifferentWorker(reason);
return RetryByReschedule(reason);
case RetryReason.OutputWithNoFileAccessFailed:
case RetryReason.MismatchedMessageCount:
case RetryReason.AzureWatsonExitCode:
case RetryReason.UserSpecifiedExitCode:
return RetryOnSameWorker(reason);
return RetryInline(reason);
case RetryReason.VmExecutionError:
return RetryOnSameAndDifferentWorkers(reason);
return RetryInlineFirstThenByReschedule(reason);
case RetryReason.RemoteFallback:
return RetryByReschedule(reason);
default:
throw Contract.AssertFailure("Default not defined for RetryReason: " + reason.ToString());
}
}
/// <summary>
/// Returns retry info to retry on a different worker when the failing process was run in a VM.
/// </summary>
/// <remarks>Temporary mitigation for bug 1871707</remarks>
public static RetryInfo GetRetryInfoForVmMitigation()
{
return RetryOnDifferentWorker(RetryReason.UserSpecifiedExitCode);
}
}
/// <summary>
/// Location where retry should occur.
/// Extensions for <see cref="RetryInfo"/>.
/// </summary>
public enum RetryLocation
public static class RetryInfoExtensions
{
/// <summary>
/// Retry on Same Worker
/// Returns true if the process can be retried inline; or false otherwise, including null instance.
/// </summary>
SameWorker = 0,
public static bool CanBeRetriedInlineOrFalseIfNull(this RetryInfo @this) => @this?.CanBeRetriedInline() ?? false;
/// <summary>
/// Retry on Different Worker
/// Returns true if the process can be retried by requeuing it back to the scheduler; or false otherwise, including null instance.
/// </summary>
DifferentWorker = 1,
public static bool CanBeRetriedByRescheduleOrFalseIfNull(this RetryInfo @this) => @this?.CanBeRetriedByReschedule() ?? false;
}
/// <summary>
/// Mode or mechanism for retrying processes.
/// </summary>
public enum RetryMode
{
/// <summary>
/// Asks the pip executor to re-execute the process without requeuing (sending the process back) to the scheduler.
/// </summary>
/// <remarks>
/// This retry mode guarantees that the process will be retried on the same worker.
/// </remarks>
Inline = 0,
/// <summary>
/// Retry on Same Worker before retrying on Different Worker
/// Behavior mimics <see cref="SameWorker"/> until we reach the local retry limit, and then mimics <see cref="DifferentWorker"/>.
/// Ask the pip executor to send the process back to the scheduler so that it goes through to the scheduler queue.
/// </summary>
/// <remarks>
/// In distributed build, this retry mode can make the process re-execute on a different worker. In non distributed build,
/// the process will be retried on the same worker until reaching retry limit.
/// </remarks>
Reschedule = 1,
/// <summary>
/// Retry inline first until reaching retry limit, then reschedule.
/// </summary>
/// <remarks>
/// This mode is useful when we want to retry the process first on the same worker (guaranteed by <see cref="Inline"/>), then
/// when failure persists, retry the process on a different worker.
/// </remarks>
Both = 2,
}
/// <summary>
/// Reasons to retry failing pips
/// Reasons for retrying a failing process.
/// </summary>
public enum RetryReason
{
@ -191,41 +194,40 @@ namespace BuildXL.Processes
/// The sandboxed process may be retried due to failures caused during VM execution.
/// </summary>
VmExecutionError = 8,
/// <summary>
/// The sandboxed process may be retried due to fallback from remote execution.
/// </summary>
RemoteFallback = 9
}
/// <summary>
/// Extensions
/// Extensions for <see cref="RetryReason"/>.
/// </summary>
public static class RetryReasonExtensions
{
/// <summary>
/// Is retryable failure during the prep
/// Is retryable failure during the pre process execution.
/// </summary>
public static bool IsPrepRetryableFailure(this RetryReason? retryReason)
public static bool IsPreProcessExecRetryableFailure(this RetryReason retryReason)
{
if (retryReason == null)
{
return false;
}
switch (retryReason)
{
case RetryReason.ProcessStartFailure:
case RetryReason.TempDirectoryCleanupFailure:
return true;
default:
return false;
}
return false;
}
/// <summary>
/// Is retryable failure due to a process failure in VM or failure during prep
/// Is retryable failure due to a pre process execution failure or due to remoting (VM or AnyBuild) infrastructure error.
/// </summary>
public static bool IsPrepOrVmFailure(this RetryReason? retryReason)
{
return retryReason.IsPrepRetryableFailure() ||
retryReason == RetryReason.VmExecutionError;
}
public static bool IsPreProcessExecOrRemotingInfraFailure(this RetryReason retryReason) =>
retryReason.IsPreProcessExecRetryableFailure()
|| retryReason == RetryReason.VmExecutionError
|| retryReason == RetryReason.RemoteFallback;
/// <summary>
/// Is retryable failure due to Detours
@ -237,9 +239,25 @@ namespace BuildXL.Processes
case RetryReason.MismatchedMessageCount:
case RetryReason.OutputWithNoFileAccessFailed:
return true;
default:
return false;
}
return false;
}
}
/// <summary>
/// Extensions for <see cref="RetryMode"/>.
/// </summary>
public static class RetryLocationExtensions
{
/// <summary>
/// Checks if retry can be done inline by the pip executor.
/// </summary>
public static bool CanBeRetriedInline(this RetryMode @this) => @this == RetryMode.Inline || @this == RetryMode.Both;
/// <summary>
/// Checks if retry can be done by requeuing the process to the scheduler.
/// </summary>
public static bool CanBeRetriedByReschedule(this RetryMode @this) => @this == RetryMode.Reschedule || @this == RetryMode.Both;
}
}

10
Public/Src/Engine/Processes/SandboxedProcessPipExecutionResult.cs
Просмотреть файл

@ -392,12 +392,12 @@ namespace BuildXL.Processes
Contract.Requires(numberOfWarnings >= 0);
Contract.Requires(containerConfiguration != null);
Contract.Requires(retryInfo == null || status != SandboxedProcessPipExecutionStatus.Succeeded);
// Protect against invalid combinations of RetryLocation and RetryReason
Contract.Requires(!RetryInfo.RetryAbleOnSameWorker(retryInfo) || retryInfo.RetryReason != RetryReason.ResourceExhaustion);
Contract.Requires(!RetryInfo.RetryAbleOnSameWorker(retryInfo) || retryInfo.RetryReason != RetryReason.ProcessStartFailure);
Contract.Requires(!RetryInfo.RetryAbleOnSameWorker(retryInfo) || retryInfo.RetryReason != RetryReason.TempDirectoryCleanupFailure);
Contract.Requires(!RetryInfo.RetryAbleOnSameWorker(retryInfo) || retryInfo.RetryReason != RetryReason.StoppedWorker);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.ResourceExhaustion);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.ProcessStartFailure);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.TempDirectoryCleanupFailure);
Contract.Requires(!retryInfo.CanBeRetriedInlineOrFalseIfNull() || retryInfo.RetryReason != RetryReason.StoppedWorker);
Status = status;
ObservedFileAccesses = observedFileAccesses;

82
Public/Src/Engine/Processes/SandboxedProcessPipExecutor.cs
Просмотреть файл

@ -227,8 +227,9 @@ namespace BuildXL.Processes
/// </summary>
public static readonly string StdOutputsDirNameInLog = "StdOutputs";
private bool m_forceExecuteLocally = false;
private readonly ProcessRunLocation m_runLocation = ProcessRunLocation.Default;
private readonly RemoteSandboxedProcessData.Builder m_remoteSbDataBuilder;
/// <summary>
/// Creates an executor for a process pip. Execution can then be started with <see cref="RunAsync" />.
/// </summary>
@ -262,7 +263,8 @@ namespace BuildXL.Processes
IReadOnlyDictionary<AbsolutePath, IReadOnlyCollection<FileArtifactWithAttributes>> staleOutputsUnderSharedOpaqueDirectories = null,
PluginManager pluginManager = null,
ISandboxFileSystemView sandboxFileSystemView = null,
IPipGraphFileSystemView pipGraphFileSystemView = null)
IPipGraphFileSystemView pipGraphFileSystemView = null,
ProcessRunLocation runLocation = ProcessRunLocation.Default)
{
Contract.Requires(pip != null);
Contract.Requires(context != null);
@ -405,6 +407,12 @@ namespace BuildXL.Processes
m_staleOutputsUnderSharedOpaqueDirectoriesToBeDeletedInVM = new List<string>();
m_fileSystemView = sandboxFileSystemView;
m_pipGraphFileSystemView = pipGraphFileSystemView;
m_runLocation = runLocation;
if (runLocation == ProcessRunLocation.Remote)
{
m_remoteSbDataBuilder = new RemoteSandboxedProcessData.Builder(m_pathTable);
}
}
/// <inheritdoc />
@ -887,21 +895,16 @@ namespace BuildXL.Processes
}
}
private bool SandboxedProcessNeedsExecuteRemote => m_sandboxConfig.RemoteAllProcesses;
private bool SandboxedProcessShouldExecuteRemote => m_runLocation == ProcessRunLocation.Remote;
private bool SandboxedProcessNeedsExecuteExternal =>
SandboxedProcessNeedsExecuteRemote
SandboxedProcessShouldExecuteRemote
|| (// Execution mode is external
m_sandboxConfig.AdminRequiredProcessExecutionMode.ExecuteExternal()
// Only pip that requires admin privilege.
&& m_pip.RequiresAdmin);
private bool ShouldSandboxedProcessExecuteExternal =>
SandboxedProcessNeedsExecuteExternal
// Container is disabled.
&& !m_containerConfiguration.IsIsolationEnabled
// Local execution is not enforced when process needs to execute remotely.
&& (!SandboxedProcessNeedsExecuteRemote || !m_forceExecuteLocally);
private bool ShouldSandboxedProcessExecuteExternal => SandboxedProcessNeedsExecuteExternal && !m_containerConfiguration.IsIsolationEnabled;
private bool ShouldSandboxedProcessExecuteInVm =>
ShouldSandboxedProcessExecuteExternal
@ -1112,7 +1115,7 @@ namespace BuildXL.Processes
string externalSandboxedProcessDirectory = m_layoutConfiguration.ExternalSandboxedProcessDirectory.ToString(m_pathTable);
if (SandboxedProcessNeedsExecuteRemote)
if (SandboxedProcessShouldExecuteRemote)
{
PopulateRemoteSandboxedProcessData(info);
@ -1165,7 +1168,7 @@ namespace BuildXL.Processes
private void PopulateRemoteSandboxedProcessData(SandboxedProcessInfo info)
{
if (!SandboxedProcessNeedsExecuteRemote)
if (!SandboxedProcessShouldExecuteRemote)
{
return;
}
@ -1173,19 +1176,19 @@ namespace BuildXL.Processes
// <see cref="RemoteSandboxedProcessData"/> for the reason why one should not add output directories
// when populating remote data.
var tempDirectories = new HashSet<string>(OperatingSystemHelper.PathComparer);
Contract.Assert(m_remoteSbDataBuilder != null);
if (m_pip.TempDirectory.IsValid)
{
tempDirectories.Add(m_pip.TempDirectory.ToString(m_pathTable));
m_remoteSbDataBuilder.AddTempDirectory(m_pip.TempDirectory);
}
foreach (var tempDir in m_pip.AdditionalTempDirectories)
{
tempDirectories.Add(tempDir.ToString(m_pathTable));
m_remoteSbDataBuilder.AddTempDirectory(tempDir);
}
info.RemoteSandboxedProcessData = new RemoteSandboxedProcessData(tempDirectories.ToList());
info.RemoteSandboxedProcessData = m_remoteSbDataBuilder.Build();
}
private void PopulateExternalVMSandboxedProcessData(SandboxedProcessInfo info)
@ -1253,17 +1256,10 @@ namespace BuildXL.Processes
{
if (process is RemoteSandboxedProcess remoteSandboxedProcess && remoteSandboxedProcess.ShouldRunLocally == true)
{
m_forceExecuteLocally = true;
if (!SetMessageCountSemaphoreIfRequested())
{
return SandboxedProcessPipExecutionResult.PreparationFailure();
}
return await RunAsync(
cancellationToken,
remoteSandboxedProcess.SandboxedProcessInfo.SandboxConnection,
remoteSandboxedProcess.SandboxedProcessInfo.SidebandWriter);
return SandboxedProcessPipExecutionResult.FailureButRetryAble(
SandboxedProcessPipExecutionStatus.ExecutionFailed,
RetryInfo.GetDefault(RetryReason.RemoteFallback),
primaryProcessTimes: result.PrimaryProcessTimes);
}
int exitCode = externalSandboxedProcess.ExitCode ?? -1;
@ -2231,7 +2227,7 @@ namespace BuildXL.Processes
m_sandboxConfig.LogObservedFileAccesses
// When sandboxed process needs to be remoted, the remoting infrastructure, like AnyBuild, typically requires all
// reported file accesses.
|| SandboxedProcessNeedsExecuteRemote;
|| SandboxedProcessShouldExecuteRemote;
m_fileAccessManifest.BreakOnUnexpectedAccess = m_sandboxConfig.BreakOnUnexpectedFileAccess;
m_fileAccessManifest.FailUnexpectedFileAccesses = m_sandboxConfig.FailUnexpectedFileAccesses;
m_fileAccessManifest.ReportProcessArgs = m_sandboxConfig.LogProcesses;
@ -2280,9 +2276,7 @@ namespace BuildXL.Processes
return true;
}
if ((!m_pip.RequiresAdmin || m_sandboxConfig.AdminRequiredProcessExecutionMode == AdminRequiredProcessExecutionMode.Internal)
&& (!m_sandboxConfig.RemoteAllProcesses || m_forceExecuteLocally)
&& !OperatingSystemHelper.IsUnixOS)
if (OperatingSystemHelper.IsWindowsOS && !SandboxedProcessNeedsExecuteExternal)
{
// Semaphore names don't allow '\\' chars.
if (!m_fileAccessManifest.SetMessageCountSemaphore(m_detoursFailuresFile.Replace('\\', '_')))
@ -2296,15 +2290,23 @@ namespace BuildXL.Processes
}
private void AddUntrackedScopeToManifest(AbsolutePath path, FileAccessManifest manifest = null) => (manifest ?? m_fileAccessManifest).AddScope(
path,
mask: m_excludeReportAccessMask,
values: FileAccessPolicy.AllowAll | FileAccessPolicy.AllowRealInputTimestamps);
private void AddUntrackedScopeToManifest(AbsolutePath path, FileAccessManifest manifest = null)
{
(manifest ?? m_fileAccessManifest).AddScope(
path,
mask: m_excludeReportAccessMask,
values: FileAccessPolicy.AllowAll | FileAccessPolicy.AllowRealInputTimestamps);
m_remoteSbDataBuilder?.AddUntrackedScope(path);
}
private void AddUntrackedPathToManifest(AbsolutePath path, FileAccessManifest manifest = null) => (manifest ?? m_fileAccessManifest).AddPath(
path,
mask: m_excludeReportAccessMask,
values: FileAccessPolicy.AllowAll | FileAccessPolicy.AllowRealInputTimestamps);
private void AddUntrackedPathToManifest(AbsolutePath path, FileAccessManifest manifest = null)
{
(manifest ?? m_fileAccessManifest).AddPath(
path,
mask: m_excludeReportAccessMask,
values: FileAccessPolicy.AllowAll | FileAccessPolicy.AllowRealInputTimestamps);
m_remoteSbDataBuilder?.AddUntrackedPath(path);
}
private void AllowCreateDirectoryForDirectoriesOnPath(AbsolutePath path, HashSet<AbsolutePath> processedPaths, bool startWithParent = true)
{

7
Public/Src/Engine/Scheduler/Distribution/LocalWorker.cs
Просмотреть файл

@ -20,7 +20,7 @@ namespace BuildXL.Scheduler.Distribution
/// <summary>
/// Defines an in-process worker.
/// </summary>
public sealed class LocalWorker : Worker
public class LocalWorker : Worker
{
/// <summary>
/// Set of pips that are currently executing. Executing here means running under PipExecutor.
@ -76,7 +76,7 @@ namespace BuildXL.Scheduler.Distribution
public LocalWorker(IScheduleConfiguration scheduleConfig, IPipQueue pipQueue, IDetoursEventListener detoursListener, PipExecutionContext context)
: base(workerId: 0, name: "#0 (Local)", context: context)
{
TotalProcessSlots = scheduleConfig.MaxProcesses;
TotalProcessSlots = scheduleConfig.EffectiveMaxProcesses;
TotalCacheLookupSlots = scheduleConfig.MaxCacheLookup;
TotalLightSlots = scheduleConfig.MaxLightProcesses;
TotalMaterializeInputSlots = scheduleConfig.MaxMaterialize;
@ -155,7 +155,8 @@ namespace BuildXL.Scheduler.Distribution
fingerprint,
processIdListener: UpdateCurrentlyRunningPipsCount,
expectedMemoryCounters: processRunnable.ExpectedMemoryCounters.Value,
detoursEventListener: m_detoursListener);
detoursEventListener: m_detoursListener,
runLocation: processRunnable.RunLocation);
processRunnable.SetExecutionResult(executionResult);
Unit ignore;

146
Public/Src/Engine/Scheduler/Distribution/LocalWorkerWithRemoting.cs Normal file
Просмотреть файл

@ -0,0 +1,146 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Pips.Operations;
using BuildXL.Processes;
using BuildXL.Utilities;
using BuildXL.Utilities.Configuration;
namespace BuildXL.Scheduler.Distribution
{
/// <summary>
/// Local worker with process remoting ability via AnyBuild.
/// </summary>
public class LocalWorkerWithRemoting : LocalWorker
{
private readonly SemaphoreSlim m_localExecutionSemaphore;
private readonly int m_remotingThreshold;
private readonly HashSet<StringId> m_processCanRunRemoteTags;
private readonly HashSet<StringId> m_processMustRunLocalTags;
private int m_currentRunLocalCount = 0;
private int m_currentRunRemoteCount = 0;
private int m_totalRunRemote = 0;
private int m_totalRunLocally = 0;
private int m_totalRemoteFallbackRetryLocally = 0;
/// <summary>
/// The number of processes currently being executed remotely.
/// </summary>
public int CurrentRunRemoteCount => Volatile.Read(ref m_currentRunRemoteCount);
/// <summary>
/// Total number of processes that have been executed remotely.
/// </summary>
public int TotalRunRemote => Volatile.Read(ref m_totalRunRemote);
/// <summary>
/// Total number of processes that ran locally.
/// </summary>
public int TotalRunLocally => Volatile.Read(ref m_totalRunLocally);
/// <summary>
/// Total number of remoted processes that were retried to run locally as fallback.
/// </summary>
public int TotalRemoteFallbackRetryLocally => Volatile.Read(ref m_totalRemoteFallbackRetryLocally);
private StringTable StringTable => PipExecutionContext.StringTable;
private readonly ISandboxConfiguration m_sandboxConfig;
/// <summary>
/// Constructor.
/// </summary>
public LocalWorkerWithRemoting(
IScheduleConfiguration scheduleConfig,
ISandboxConfiguration sandboxConfig,
IPipQueue pipQueue,
IDetoursEventListener detoursListener,
PipExecutionContext pipExecutionContext)
: base(scheduleConfig, pipQueue, detoursListener, pipExecutionContext)
{
m_localExecutionSemaphore = new SemaphoreSlim(scheduleConfig.MaxProcesses, scheduleConfig.MaxProcesses);
m_remotingThreshold = (int)(scheduleConfig.MaxProcesses * scheduleConfig.RemotingThresholdMultiplier);
m_processCanRunRemoteTags = scheduleConfig.ProcessCanRunRemoteTags.Select(t => StringTable.AddString(t)).ToHashSet();
m_processMustRunLocalTags = scheduleConfig.ProcessMustRunLocalTags.Select(t => StringTable.AddString(t)).ToHashSet();
m_sandboxConfig = sandboxConfig;
}
/// <inheritdoc />
public override async Task<ExecutionResult> ExecuteProcessAsync(ProcessRunnablePip processRunnable)
{
if (processRunnable.IsLight)
{
// Light process does not need remoting, and does not need a slot.
return await base.ExecuteProcessAsync(processRunnable);
}
// Assume that the process is going to run locally (prefer local).
int runLocalCount = Interlocked.Increment(ref m_currentRunLocalCount);
// Run local criteria:
// - the process is forced to run locally, i.e., run location == ProcessRunLocation.Local, or
// - the process requires an admin privilege, or
// - the user specifies "MustRunLocal" tag, and the process has that tag, or
// - the user specifies "CanRunRemote" tag, and the process does not have that tag, or
// - the number of processes that are running locally (or are going to run locally)
// is below a threshold.
//
// When the process requires an admin privilege, then most likely it has to run in a VM hosted
// by the local worker. Thus, the parallelism of running such process should be the same as running
// the process on the local worker.
if (processRunnable.RunLocation == ProcessRunLocation.Local
|| ProcessRequiresAdminPrivilege(processRunnable.Process)
|| (ExistTags(m_processMustRunLocalTags) && HasTag(processRunnable.Process, m_processMustRunLocalTags))
|| (ExistTags(m_processCanRunRemoteTags) && !HasTag(processRunnable.Process, m_processCanRunRemoteTags))
|| runLocalCount <= m_remotingThreshold)
{
await m_localExecutionSemaphore.WaitAsync();
Interlocked.Increment(ref m_totalRunLocally);
if (processRunnable.ExecutionResult?.RetryInfo?.RetryReason == RetryReason.RemoteFallback)
{
Interlocked.Increment(ref m_totalRemoteFallbackRetryLocally);
}
try
{
return await base.ExecuteProcessAsync(processRunnable);
}
finally
{
m_localExecutionSemaphore.Release();
Interlocked.Decrement(ref m_currentRunLocalCount);
}
}
else
{
// Retract the assumption that the process is going to run locally.
Interlocked.Decrement(ref m_currentRunLocalCount);
processRunnable.RunLocation = ProcessRunLocation.Remote;
Interlocked.Increment(ref m_currentRunRemoteCount);
Interlocked.Increment(ref m_totalRunRemote);
try
{
return await base.ExecuteProcessAsync(processRunnable);
}
finally
{
Interlocked.Decrement(ref m_currentRunRemoteCount);
}
}
}
private bool ExistTags(HashSet<StringId> tags) => tags != null && tags.Count > 0;
private bool HasTag(Process process, HashSet<StringId> tags) => ExistTags(tags) && process.Tags.Any(t => tags.Contains(t));
private bool ProcessRequiresAdminPrivilege(Process process) => process.RequiresAdmin && m_sandboxConfig.AdminRequiredProcessExecutionMode.ExecuteExternal();
}
}

48
Public/Src/Engine/Scheduler/Distribution/Worker.cs
Просмотреть файл

@ -20,7 +20,6 @@ using BuildXL.Utilities.Collections;
using BuildXL.Utilities.Configuration;
using BuildXL.Utilities.Tasks;
using BuildXL.Utilities.Threading;
using BuildXL.Utilities.Tracing;
using static BuildXL.Utilities.FormattableStringEx;
namespace BuildXL.Scheduler.Distribution
@ -105,10 +104,7 @@ namespace BuildXL.Scheduler.Distribution
/// </summary>
public virtual int TotalProcessSlots
{
get
{
return Volatile.Read(ref m_totalProcessSlots);
}
get => Volatile.Read(ref m_totalProcessSlots);
protected set
{
@ -146,15 +142,9 @@ namespace BuildXL.Scheduler.Distribution
/// </summary>
public int TotalLightSlots
{
get
{
return Volatile.Read(ref m_totalLightSlots);
}
get => Volatile.Read(ref m_totalLightSlots);
protected set
{
Volatile.Write(ref m_totalLightSlots, value);
}
protected set => Volatile.Write(ref m_totalLightSlots, value);
}
/// <summary>
@ -174,10 +164,7 @@ namespace BuildXL.Scheduler.Distribution
/// </summary>
public int? TotalRamMb
{
get
{
return m_totalMemoryMb;
}
get => m_totalMemoryMb;
set
{
@ -298,10 +285,7 @@ namespace BuildXL.Scheduler.Distribution
/// </summary>
public virtual WorkerNodeStatus Status
{
get
{
return m_status;
}
get => m_status;
set
{
@ -313,24 +297,12 @@ namespace BuildXL.Scheduler.Distribution
/// <summary>
/// Whether the worker become available at any time
/// </summary>
public virtual bool EverAvailable
{
get
{
return true;
}
}
public virtual bool EverAvailable => true;
/// <summary>
/// The number of the build requests waiting to be sent
/// </summary>
public virtual int WaitingBuildRequestsCount
{
get
{
return 0;
}
}
public virtual int WaitingBuildRequestsCount => 0;
/// <summary>
/// Gets the name of the worker
@ -347,6 +319,11 @@ namespace BuildXL.Scheduler.Distribution
/// </summary>
public WorkerPipStateManager.Snapshot PipStateSnapshot { get; }
/// <summary>
/// Pip execution context.
/// </summary>
protected PipExecutionContext PipExecutionContext { init; get; }
private readonly OperationKind m_workerOperationKind;
/// <summary>
@ -362,6 +339,7 @@ namespace BuildXL.Scheduler.Distribution
m_workerOperationKind = OperationKind.Create("Worker " + Name);
DrainCompletion = TaskSourceSlim.Create<bool>();
PipExecutionContext = context;
InitSemaphores(context);
}

9
Public/Src/Engine/Scheduler/ExecutionResult.cs
Просмотреть файл

@ -713,9 +713,10 @@ namespace BuildXL.Scheduler
SandboxedProcessPipExecutionResult processResult = m_unsealedState.ExecutionResult;
if (processResult != null &&
processResult.Status != SandboxedProcessPipExecutionStatus.PreparationFailed &&
!(processResult.RetryInfo?.RetryReason).IsPrepOrVmFailure())
if (processResult != null
&& processResult.Status != SandboxedProcessPipExecutionStatus.PreparationFailed
&& (processResult.RetryInfo == null
|| !(processResult.RetryInfo.RetryReason).IsPreProcessExecOrRemotingInfraFailure()))
{
if (!(processResult.Status == SandboxedProcessPipExecutionStatus.Succeeded ||
processResult.Status == SandboxedProcessPipExecutionStatus.ExecutionFailed ||
@ -725,7 +726,7 @@ namespace BuildXL.Scheduler
processResult.RetryInfo?.RetryReason == RetryReason.OutputWithNoFileAccessFailed ||
processResult.RetryInfo?.RetryReason == RetryReason.MismatchedMessageCount))
{
string retryReason = processResult.RetryInfo != null ? $", Retry Reason: {processResult.RetryInfo.RetryReason}, Retry Location: {processResult.RetryInfo.RetryLocation}" : "";
string retryReason = processResult.RetryInfo != null ? $", Retry Reason: {processResult.RetryInfo.RetryReason}, Retry Location: {processResult.RetryInfo.RetryMode}" : "";
Contract.Assert(false, "Invalid execution status: " + processResult.Status + retryReason);
}

49
Public/Src/Engine/Scheduler/PipExecutor.cs
Просмотреть файл

@ -1204,6 +1204,7 @@ namespace BuildXL.Scheduler
/// <param name="processIdListener">Callback to call when the process is actually started (PID is passed to it) and when the process exited (negative PID is passed to it)</param>
/// <param name="expectedMemoryCounters">the expected memory counters for the process in megabytes</param>
/// <param name="detoursEventListener">Detours listener to collect detours reported accesses. For tests only</param>
/// <param name="runLocation">Location for running the process.</param>
/// <returns>A task that returns the execution result when done</returns>
public static async Task<ExecutionResult> ExecuteProcessAsync(
OperationContext operationContext,
@ -1215,7 +1216,8 @@ namespace BuildXL.Scheduler
ContentFingerprint? fingerprint,
Action<int> processIdListener = null,
ProcessMemoryCounters expectedMemoryCounters = default(ProcessMemoryCounters),
IDetoursEventListener detoursEventListener = null)
IDetoursEventListener detoursEventListener = null,
ProcessRunLocation runLocation = ProcessRunLocation.Default)
{
var context = environment.Context;
var counters = environment.Counters;
@ -1263,8 +1265,20 @@ namespace BuildXL.Scheduler
pip,
expectedMemoryCounters,
allowResourceBasedCancellation,
async (resourceScope) => { return await ExecutePipAndHandleRetryAsync(resourceScope,
operationContext, pip, expectedMemoryCounters, environment, state, processIdListener, detoursEventListener, start); });
async (resourceScope) =>
{
return await ExecutePipAndHandleRetryAsync(
resourceScope,
operationContext,
pip,
expectedMemoryCounters,
environment,
state,
processIdListener,
detoursEventListener,
runLocation,
start);
});
processExecutionResult.ReportSandboxedExecutionResult(executionResult);
LogSubPhaseDuration(operationContext, pip, SandboxedProcessCounters.PipExecutorPhaseReportingExeResult, DateTime.UtcNow.Subtract(start));
@ -1278,7 +1292,7 @@ namespace BuildXL.Scheduler
// We may have some violations reported already (outright denied by the sandbox manifest).
FileAccessReportingContext fileAccessReportingContext = executionResult.UnexpectedFileAccesses;
if (RetryInfo.RetryAbleOnDifferentWorker(executionResult.RetryInfo))
if (executionResult.RetryInfo.CanBeRetriedByRescheduleOrFalseIfNull())
{
// No post processing for retryable pips
processExecutionResult.SetResult(operationContext, PipResultStatus.Canceled, executionResult.RetryInfo);
@ -1334,7 +1348,7 @@ namespace BuildXL.Scheduler
executionResult.RetryInfo?.RetryReason == RetryReason.MismatchedMessageCount ||
executionResult.RetryInfo?.RetryReason == RetryReason.AzureWatsonExitCode)
{
Contract.Assert(operationContext.LoggingContext.ErrorWasLogged, I($"Error should have been logged for failures after multiple retries on {executionResult.RetryInfo?.RetryLocation.ToString()} due to '{executionResult.RetryInfo?.RetryReason.ToString()}'"));
Contract.Assert(operationContext.LoggingContext.ErrorWasLogged, I($"Error should have been logged for failures after multiple retries on {executionResult.RetryInfo?.RetryMode.ToString()} due to '{executionResult.RetryInfo?.RetryReason.ToString()}'"));
}
Contract.Assert(executionResult.UnexpectedFileAccesses != null, "Success / ExecutionFailed provides all execution-time fields");
@ -1565,6 +1579,7 @@ namespace BuildXL.Scheduler
PipExecutionState.PipScopeState state,
Action<int> processIdListener,
IDetoursEventListener detoursEventListener,
ProcessRunLocation runLocation,
DateTime start)
{
var context = environment.Context;
@ -1770,7 +1785,8 @@ namespace BuildXL.Scheduler
reparsePointResolver: environment.ReparsePointAccessResolver,
staleOutputsUnderSharedOpaqueDirectories: staleDynamicOutputs,
pluginManager: environment.PluginManager,
pipGraphFileSystemView: environment.PipGraphView);
pipGraphFileSystemView: environment.PipGraphView,
runLocation: runLocation);
resourceScope.RegisterQueryRamUsageMb(
() =>
@ -1861,9 +1877,9 @@ namespace BuildXL.Scheduler
}
}
if (result.RetryInfo?.RetryLocation == RetryLocation.DifferentWorker)
if (result.RetryInfo?.RetryMode == RetryMode.Reschedule)
{
Logger.Log.PipProcessToBeRetriedOnDifferentWorker(operationContext,
Logger.Log.PipProcessToBeRetriedByReschedule(operationContext,
processDescription, result.RetryInfo.RetryReason.ToString());
}
@ -1880,20 +1896,19 @@ namespace BuildXL.Scheduler
continue;
}
if (RetryInfo.RetryAbleOnSameWorker(result.RetryInfo))
if (result.RetryInfo.CanBeRetriedInlineOrFalseIfNull())
{
if (remainingInternalSandboxedProcessExecutionFailureRetries <= 0)
{
if (result.RetryInfo.RetryLocation == RetryLocation.SameWorker)
if (result.RetryInfo.RetryMode == RetryMode.Inline)
{
// Log errors for Retry cases on the SameWorker which have reached their local retry limit
LogRetryOnSameWorkerErrors(result.RetryInfo.RetryReason, operationContext, pip, processDescription);
// Log errors for inline retry on the same worker which have reached their local retry limit
LogRetryInlineErrors(result.RetryInfo.RetryReason, operationContext, pip, processDescription);
break;
}
else // Case: RetryLocation.Both
{
Logger.Log.PipProcessToBeRetriedOnDifferentWorker(operationContext,
processDescription, result.RetryInfo.RetryReason.ToString());
Logger.Log.PipProcessToBeRetriedByReschedule(operationContext, processDescription, result.RetryInfo.RetryReason.ToString());
break;
}
}
@ -1907,7 +1922,7 @@ namespace BuildXL.Scheduler
--remainingInternalSandboxedProcessExecutionFailureRetries;
Logger.Log.PipProcessToBeRetriedOnSameWorker(operationContext,
Logger.Log.PipProcessRetriedInline(operationContext,
InternalSandboxedProcessExecutionFailureRetryCountMax - remainingInternalSandboxedProcessExecutionFailureRetries,
InternalSandboxedProcessExecutionFailureRetryCountMax,
processDescription, result.RetryInfo.RetryReason.ToString());
@ -1918,10 +1933,12 @@ namespace BuildXL.Scheduler
{
Contract.Assert(false, "Unexpected result error type.");
}
continue;
}
// Just break the loop below. The result is already set properly.
}
break;
}
@ -1991,7 +2008,7 @@ namespace BuildXL.Scheduler
return false; // Unhandled case, needs to be handled by the caller
}
private static void LogRetryOnSameWorkerErrors(RetryReason retryReason, OperationContext operationContext, Process pip, string processDescription)
private static void LogRetryInlineErrors(RetryReason retryReason, OperationContext operationContext, Process pip, string processDescription)
{
switch (retryReason)
{

15
Public/Src/Engine/Scheduler/PipExecutorCounter.cs
Просмотреть файл

@ -1286,6 +1286,11 @@ namespace BuildXL.Scheduler
/// </summary>
VmExecutionRetriesCount,
/// <summary>
/// Total number of processes that were retried locally after running remotely in due to some remoting failure.
/// </summary>
TotalRemoteFallbackRetries,
/// <summary>
/// Number of times non-existent directory probes for paths under opaque directories
/// re-classified as existing directory probes.
@ -1303,6 +1308,16 @@ namespace BuildXL.Scheduler
/// </summary>
[CounterType(CounterType.Stopwatch)]
FileContentManagerEnumerateOutputDirectoryHashArtifacts,
/// <summary>
/// Total number of processes that ran remotely.
/// </summary>
TotalRunRemoteProcesses,
/// <summary>
/// Total number of processes that ran locally on worker capable of remoting.
/// </summary>
TotalRunLocallyProcessesOnRemotingWorker
}
/// <summary>

4
Public/Src/Engine/Scheduler/PipQueue.cs
Просмотреть файл

@ -179,7 +179,7 @@ namespace BuildXL.Scheduler
{DispatcherKind.ChooseWorkerLight, m_chooseWorkerLightQueue },
{DispatcherKind.CacheLookup, new DispatcherQueue(this, m_scheduleConfig.MaxCacheLookup)},
{DispatcherKind.ChooseWorkerCpu, m_chooseWorkerCpuQueue},
{DispatcherKind.CPU, new DispatcherQueue(this, m_scheduleConfig.MaxProcesses, useWeight: true)},
{DispatcherKind.CPU, new DispatcherQueue(this, m_scheduleConfig.EffectiveMaxProcesses, useWeight: true)},
{DispatcherKind.Materialize, new DispatcherQueue(this, m_scheduleConfig.MaxMaterialize)},
{DispatcherKind.Light, new DispatcherQueue(this, m_scheduleConfig.MaxLightProcesses)},
{DispatcherKind.SealDirs, new DispatcherQueue(this, m_scheduleConfig.MaxSealDirs)},
@ -193,7 +193,7 @@ namespace BuildXL.Scheduler
m_scheduleConfig.MaxChooseWorkerCacheLookup,
m_scheduleConfig.MaxCacheLookup,
m_scheduleConfig.MaxChooseWorkerCpu,
m_scheduleConfig.MaxProcesses,
m_scheduleConfig.EffectiveMaxProcesses,
m_scheduleConfig.MaxMaterialize,
m_scheduleConfig.MaxLightProcesses,
m_scheduleConfig.OrchestratorCacheLookupMultiplier.ToString(),

6
Public/Src/Engine/Scheduler/ProcessRunnablePip.cs
Просмотреть файл

@ -6,6 +6,7 @@ using System.Collections.Generic;
using System.Threading.Tasks;
using BuildXL.Pips;
using BuildXL.Pips.Operations;
using BuildXL.Processes;
using BuildXL.Scheduler.Tracing;
using BuildXL.Utilities;
using BuildXL.Utilities.Collections;
@ -69,6 +70,11 @@ namespace BuildXL.Scheduler
/// </summary>
public IReadOnlyCollection<AbsolutePath> ChangeAffectedInputs { get; set; }
/// <summary>
/// Run location of the process.
/// </summary>
public ProcessRunLocation RunLocation { get; set; } = ProcessRunLocation.Default;
private readonly int m_weightBasedOnHistoricCpuUsage;
internal ProcessRunnablePip(

37
Public/Src/Engine/Scheduler/Scheduler.cs
Просмотреть файл

@ -429,7 +429,7 @@ namespace BuildXL.Scheduler
{
int availableRemoteWorkersCount = AvailableRemoteWorkersCount;
int targetProcessSlots = m_scheduleConfiguration.MaxProcesses;
int targetProcessSlots = m_scheduleConfiguration.EffectiveMaxProcesses;
int targetCacheLookupSlots = m_scheduleConfiguration.MaxCacheLookup;
int newProcessSlots;
@ -1241,7 +1241,7 @@ namespace BuildXL.Scheduler
m_schedulerCancellationTokenSource = new CancellationTokenSource();
m_servicePipTracker = new ServicePipTracker(Context);
m_servicePipTracker = new ServicePipTracker(context);
m_serviceManager = new SchedulerServiceManager(graph, context, m_servicePipTracker);
m_pipFragmentRenderer = this.CreatePipFragmentRenderer();
m_ipcProvider = new IpcProviderWithMemoization(
@ -1282,7 +1282,9 @@ namespace BuildXL.Scheduler
}
m_testHooks = testHooks;
LocalWorker = new LocalWorker(m_scheduleConfiguration, pipQueue, m_testHooks?.DetoursListener, Context);
LocalWorker = m_scheduleConfiguration.EnableProcessRemoting
? new LocalWorkerWithRemoting(m_scheduleConfiguration, m_configuration.Sandbox, pipQueue, m_testHooks?.DetoursListener, context)
: new LocalWorker(m_scheduleConfiguration, pipQueue, m_testHooks?.DetoursListener, context);
m_workers = new List<Worker> { LocalWorker };
m_statusSnapshotLastUpdated = DateTime.UtcNow;
@ -1761,6 +1763,8 @@ namespace BuildXL.Scheduler
public SchedulerPerformanceInfo LogStats(LoggingContext loggingContext, [CanBeNull] BuildSummary buildSummary)
{
Dictionary<string, long> statistics = new Dictionary<string, long>();
LocalWorkerWithRemoting localWorkerWithRemoting = LocalWorker as LocalWorkerWithRemoting;
lock (m_statusLock)
{
m_fileContentManager.LogStats(loggingContext);
@ -1942,6 +1946,10 @@ namespace BuildXL.Scheduler
PipExecutionCounters.AddToCounter(PipExecutorCounter.MaxPathSetsDownloadedForMiss, cacheLookupPerfInfosForMisses.Max(a => a?.NumPathSetsDownloaded) ?? -1);
PipExecutionCounters.AddToCounter(PipExecutorCounter.MinPathSetsDownloadedForMiss, cacheLookupPerfInfosForMisses.Min(a => a?.NumPathSetsDownloaded) ?? -1);
PipExecutionCounters.AddToCounter(PipExecutorCounter.TotalRunRemoteProcesses, localWorkerWithRemoting != null ? localWorkerWithRemoting.TotalRunRemote : 0);
PipExecutionCounters.AddToCounter(PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker, localWorkerWithRemoting != null ? localWorkerWithRemoting.TotalRunLocally : 0);
PipExecutionCounters.AddToCounter(PipExecutorCounter.TotalRemoteFallbackRetries, localWorkerWithRemoting != null ? localWorkerWithRemoting.TotalRemoteFallbackRetryLocally : 0);
var currentTime = DateTime.UtcNow;
var earlyReleaseSavingDurationMs = Workers.Where(a => a.WorkerEarlyReleasedTime != null).Select(a => (currentTime - a.WorkerEarlyReleasedTime.Value).TotalMilliseconds).Sum();
PipExecutionCounters.AddToCounter(PipExecutorCounter.RemoteWorker_EarlyReleaseSavingDurationMs, (long)earlyReleaseSavingDurationMs);
@ -2421,7 +2429,8 @@ namespace BuildXL.Scheduler
copyFileDone: copyFileStats.DoneCount,
copyFileNotDone: copyFileStats.Total - copyFileStats.DoneCount - copyFileStats.IgnoredCount,
writeFileDone: writeFileStats.DoneCount,
writeFileNotDone: writeFileStats.Total - writeFileStats.DoneCount - writeFileStats.IgnoredCount);
writeFileNotDone: writeFileStats.Total - writeFileStats.DoneCount - writeFileStats.IgnoredCount,
procsRemoted: LocalWorker is LocalWorkerWithRemoting remoteLocal ? remoteLocal.CurrentRunRemoteCount : 0);
}
// Number of process pips that are not completed yet.
@ -2557,7 +2566,8 @@ namespace BuildXL.Scheduler
long copyFileDone,
long copyFileNotDone,
long writeFileDone,
long writeFileNotDone)
long writeFileNotDone,
long procsRemoted)
{
// Noop if no process information is included. This can happen for the last status event in a build using
// incremental scheduling if it goes through the codepath where zero files changed. All other codepaths
@ -2594,7 +2604,8 @@ namespace BuildXL.Scheduler
copyFileDone,
copyFileNotDone,
writeFileDone,
writeFileNotDone);
writeFileNotDone,
procsRemoted);
}
else
{
@ -2623,7 +2634,8 @@ namespace BuildXL.Scheduler
copyFileDone,
copyFileNotDone,
writeFileDone,
writeFileNotDone);
writeFileNotDone,
procsRemoted);
}
}
@ -4419,8 +4431,8 @@ namespace BuildXL.Scheduler
// The pip was canceled due to retryable failure
if (executionResult.Result == PipResultStatus.Canceled && !IsTerminating)
{
Contract.Requires(executionResult.RetryInfo != null, $"Retry Information is required for all retry cases. IsTerminating: {m_scheduleTerminating}");
RetryReason? retryReason = executionResult.RetryInfo.RetryReason;
Contract.Assert(executionResult.RetryInfo != null, $"Retry Information is required for all retry cases. IsTerminating: {m_scheduleTerminating}");
RetryReason retryReason = executionResult.RetryInfo.RetryReason;
if (worker.IsLocal)
{
@ -4454,7 +4466,7 @@ namespace BuildXL.Scheduler
Logger.Log.PipRetryDueToLowMemory(operationContext, processRunnable.Description, worker.DefaultWorkingSetMbPerProcess, expectedCounters.PeakWorkingSetMb, actualCounters?.PeakWorkingSetMb ?? 0);
}
}
else if (retryReason.IsPrepOrVmFailure())
else if (retryReason.IsPreProcessExecOrRemotingInfraFailure())
{
if (processRunnable.Performance.RetryCountDueToRetryableFailures == m_scheduleConfiguration.MaxRetriesDueToRetryableFailures)
{
@ -4465,6 +4477,11 @@ namespace BuildXL.Scheduler
else
{
Logger.Log.PipRetryDueToRetryableFailures(operationContext, processRunnable.Description, retryReason.ToString());
if (retryReason == RetryReason.RemoteFallback)
{
// Force local execution.
processRunnable.RunLocation = ProcessRunLocation.Local;
}
}
}
}

20
Public/Src/Engine/Scheduler/Tracing/Log.cs
Просмотреть файл

@ -1764,7 +1764,7 @@ namespace BuildXL.Scheduler.Tracing
"{procsPending} pending, {procsWaiting} waiting {procsNotIgnored} total. | " +
"All:{pipsSucceeded} succeeded, {pipsFailed} failed, {pipsSkippedDueToFailedDependencies} skipped, {pipsRunning} running, {pipsReady} ready," +
" {pipsWaiting} waiting ({pipsWaitingOnSemaphore} on semaphore), {pipsWaitingOnResources} resource paused. Services: {servicePipsRunning}." +
" LimitingResource:{limitingResource}. {perfInfoForLog}";
" LimitingResource:{limitingResource}. Remote: {procsRemoted}. {perfInfoForLog}";
private const Generators StatusGenerators = EventGenerators.LocalOnly;
private const Level StatusLevel = Level.Informational;
@ -1803,7 +1803,8 @@ namespace BuildXL.Scheduler.Tracing
long copyFileDone,
long copyFileNotDone,
long writeFileDone,
long writeFileNotDone);
long writeFileNotDone,
long procsRemoted);
[GeneratedEvent(
(ushort)LogEventId.PipStatusNonOverwriteable,
@ -1837,7 +1838,8 @@ namespace BuildXL.Scheduler.Tracing
long copyFileDone,
long copyFileNotDone,
long writeFileDone,
long writeFileNotDone);
long writeFileNotDone,
long procsRemoted);
#endregion
[GeneratedEvent(
@ -3766,22 +3768,22 @@ namespace BuildXL.Scheduler.Tracing
internal abstract void HandlePipStepOnWorkerFailed(LoggingContext loggingContext, string pipDescription, string errorMessage);
[GeneratedEvent(
(int)LogEventId.PipProcessRetriedOnSameWorker,
(int)LogEventId.PipProcessRetriedInline,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.PipExecutor,
Message = "[{pipDescription}] Pip failed due to '{reason}'. Will be retried on the same worker. Attempt {attempt} of {limit}.")]
public abstract void PipProcessToBeRetriedOnSameWorker(LoggingContext context, int attempt, int limit, string pipDescription, string reason);
Message = "[{pipDescription}] Pip failed due to '{reason}', and will be retried inline on the same worker. Attempt {attempt} of {limit}.")]
public abstract void PipProcessRetriedInline(LoggingContext context, int attempt, int limit, string pipDescription, string reason);
[GeneratedEvent(
(int)LogEventId.PipProcessRetriedOnDifferentWorker,
(int)LogEventId.PipProcessRetriedByReschedule,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
Keywords = (int)Keywords.UserMessage,
EventTask = (int)Tasks.PipExecutor,
Message = "[{pipDescription}] Pip failed due to '{reason}'. Will be retried on another worker.")]
public abstract void PipProcessToBeRetriedOnDifferentWorker(LoggingContext context, string pipDescription, string reason);
Message = "[{pipDescription}] Pip failed due to '{reason}', and will be reschduled. Rescheduling can cause the pip to run on another worker.")]
public abstract void PipProcessToBeRetriedByReschedule(LoggingContext context, string pipDescription, string reason);
[GeneratedEvent(
(ushort)LogEventId.FileContentManagerTryMaterializeFileAsyncFileArtifactAvailableLater,

4
Public/Src/Engine/Scheduler/Tracing/LogEventId.cs
Просмотреть файл

@ -486,8 +486,8 @@ namespace BuildXL.Scheduler.Tracing
// Retry Pips on Same/Different Workers
ExcessivePipRetriesDueToRetryableFailures = 14514,
PipRetryDueToRetryableFailures = 14515,
PipProcessRetriedOnSameWorker = 14516,
PipProcessRetriedOnDifferentWorker = 14517,
PipProcessRetriedInline = 14516,
PipProcessRetriedByReschedule = 14517,
FileContentManagerTryMaterializeFileAsyncFileArtifactAvailableLater = 14518,
ModuleWorkerMapping = 14519,
AddedNewWorkerToModuleAffinity = 14520,

8
Public/Src/Engine/UnitTests/ExternalToolTests/ExternalVMExecutionTests.cs
Просмотреть файл

@ -82,8 +82,8 @@ namespace ExternalToolTest.BuildXL.Scheduler
RunScheduler().AssertFailure();
AssertVerboseEventLogged(LogEventId.PipRetryDueToRetryableFailures, Configuration.Schedule.MaxRetriesDueToRetryableFailures, allowMore: true);
AssertVerboseEventLogged(LogEventId.PipProcessRetriedOnSameWorker, Configuration.Schedule.MaxRetriesDueToRetryableFailures, allowMore: true);
AssertVerboseEventLogged(LogEventId.PipProcessRetriedOnDifferentWorker, Configuration.Schedule.MaxRetriesDueToRetryableFailures, allowMore: true);
AssertVerboseEventLogged(LogEventId.PipProcessRetriedInline, Configuration.Schedule.MaxRetriesDueToRetryableFailures, allowMore: true);
AssertVerboseEventLogged(LogEventId.PipProcessRetriedByReschedule, Configuration.Schedule.MaxRetriesDueToRetryableFailures, allowMore: true);
AssertErrorEventLogged(LogEventId.ExcessivePipRetriesDueToRetryableFailures);
}
@ -109,8 +109,8 @@ namespace ExternalToolTest.BuildXL.Scheduler
RunScheduler().AssertFailure();
AssertVerboseEventLogged(LogEventId.PipRetryDueToRetryableFailures, 0, allowMore: false); // Should not be retried
AssertVerboseEventLogged(LogEventId.PipProcessRetriedOnSameWorker, 0, allowMore: false); // Should not be retried on same worker
AssertVerboseEventLogged(LogEventId.PipProcessRetriedOnDifferentWorker, 0, allowMore: false); // Should not be retried on different worker
AssertVerboseEventLogged(LogEventId.PipProcessRetriedInline, 0, allowMore: false); // Should not be retried on same worker
AssertVerboseEventLogged(LogEventId.PipProcessRetriedByReschedule, 0, allowMore: false); // Should not be retried on different worker
AssertErrorEventLogged(global::BuildXL.Processes.Tracing.LogEventId.PipProcessError);
}

117
Public/Src/Engine/UnitTests/ExternalToolTests/RemoteExecutionTests.cs Normal file
Просмотреть файл

@ -0,0 +1,117 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using BuildXL.Pips.Builders;
using BuildXL.Processes.Remoting;
using Test.BuildXL.Executables.TestProcess;
using Test.BuildXL.Scheduler;
using Test.BuildXL.TestUtilities.Xunit;
using Xunit;
using Xunit.Abstractions;
namespace ExternalToolTest.BuildXL.Scheduler
{
public sealed class RemoteExecutionTests : SchedulerIntegrationTestBase
{
public RemoteExecutionTests(ITestOutputHelper output) : base(output)
{
ShouldLogSchedulerStats = true;
Configuration.Schedule.EnableProcessRemoting = true;
RemoteSandboxedProcess.RemoteProcessFactory = new Lazy<IRemoteProcessFactory>(() => new TestRemoteProcessFactory());
}
[Fact]
public void RunSingleProcessRemotely()
{
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
RunScheduler().AssertCacheHit(process.Process.PipId);
}
[Fact]
public void RunMultipleProcessesLocallyAndRemotely()
{
Configuration.Schedule.RemotingThresholdMultiplier = 1;
Configuration.Schedule.NumOfRemoteAgentLeases = 2;
Configuration.Schedule.MaxProcesses = 1;
for (int i = 0; i < 5; ++i)
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
SchedulePipBuilder(builder);
}
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.IsTrue(result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses) > 0);
XAssert.IsTrue(result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker) > 0);
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
[Fact]
public void AllMultipleProcessesRunLocally()
{
Configuration.Schedule.RemotingThresholdMultiplier = 4;
Configuration.Schedule.NumOfRemoteAgentLeases = 2;
Configuration.Schedule.MaxProcesses = 1;
for (int i = 0; i < 5; ++i)
{
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
SchedulePipBuilder(builder);
}
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(5, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
[Fact]
public void ProcessMustRunLocalDueToTag()
{
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
// This configuration will test that must-run-local tags take precendence over can-run-remote tags.
const string MustRunLocalTag = nameof(MustRunLocalTag);
Configuration.Schedule.ProcessMustRunLocalTags = new List<string> { MustRunLocalTag };
Configuration.Schedule.ProcessCanRunRemoteTags = new List<string> { MustRunLocalTag };
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
builder.AddTags(Context.StringTable, MustRunLocalTag);
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(0, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
[Fact]
public void RemoteFallbackProcessRetryToRunLocally()
{
RemoteSandboxedProcess.RemoteProcessFactory = new Lazy<IRemoteProcessFactory>(() => new TestRemoteProcessFactory(true));
// Force run remotely.
Configuration.Schedule.RemotingThresholdMultiplier = 0.0;
ProcessBuilder builder = CreatePipBuilder(new[] { Operation.ReadFile(CreateSourceFile()), Operation.WriteFile(CreateOutputFileArtifact()) });
ProcessWithOutputs process = SchedulePipBuilder(builder);
ScheduleRunResult result = RunScheduler().AssertSuccess();
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunRemoteProcesses));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRunLocallyProcessesOnRemotingWorker));
XAssert.AreEqual(1, result.PipExecutorCounters.GetCounterValue(global::BuildXL.Scheduler.PipExecutorCounter.TotalRemoteFallbackRetries));
}
}
}

126
Public/Src/Engine/UnitTests/ExternalToolTests/TestRemoteProcess.cs Normal file
Просмотреть файл

@ -0,0 +1,126 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Diagnostics;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Processes.Remoting;
using BuildXL.Utilities;
namespace ExternalToolTest.BuildXL.Scheduler
{
/// <summary>
/// Remote process for testing.
/// </summary>
internal class TestRemoteProcess : IRemoteProcess
{
private readonly StringBuilder m_output = new();
private readonly StringBuilder m_error = new ();
private readonly RemoteCommandExecutionInfo m_processInfo;
private readonly CancellationToken m_cancellationToken;
private readonly bool m_shouldRunLocally;
private AsyncProcessExecutor m_processExecutor;
private Process m_process;
private Task<IRemoteProcessResult> m_completion;
public Task<IRemoteProcessResult> Completion => m_completion;
/// <summary>
/// Creates an instance of <see cref="TestRemoteProcess"/>.
/// </summary>
public TestRemoteProcess(
RemoteCommandExecutionInfo processInfo,
CancellationToken cancellationToken,
bool shouldRunLocally = false)
{
m_processInfo = processInfo;
m_cancellationToken = cancellationToken;
m_shouldRunLocally = shouldRunLocally;
}
/// <inheritdoc/>
public void Dispose() => m_processExecutor?.Dispose();
public Task StartAsync()
{
m_process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = m_processInfo.Command,
Arguments = m_processInfo.Args,
WorkingDirectory = m_processInfo.WorkingDirectory,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
},
EnableRaisingEvents = true
};
m_processExecutor = new AsyncProcessExecutor(
m_process,
TimeSpan.FromMilliseconds(-1), // Timeout should only be applied to the process that the external tool executes.
line =>
{
if (line != null)
{
m_output.AppendLine(line);
}
},
line =>
{
if (line != null)
{
m_error.Append(line);
}
},
$"{nameof(TestRemoteProcess)}: {m_processInfo.Command} {m_processInfo.Args}");
m_processExecutor.Start();
m_completion = WaitForCompletionAsync();
return Task.CompletedTask;
}
private async Task<IRemoteProcessResult> WaitForCompletionAsync()
{
await m_processExecutor.WaitForExitAsync();
await m_processExecutor.WaitForStdOutAndStdErrAsync();
return m_processExecutor.TimedOut || m_processExecutor.Killed || m_shouldRunLocally
? RemoteProcessResult.CreateForLocalRun()
: RemoteProcessResult.CreateFromCompletedProcess(
m_process.ExitCode,
m_output.ToString(),
m_error.ToString(),
CommandExecutionDisposition.Remoted);
}
}
internal sealed class RemoteProcessResult : IRemoteProcessResult
{
/// <inheritdoc/>
public bool ShouldRunLocally { get; private set; }
/// <inheritdoc/>
public int? ExitCode { get; private set; }
/// <inheritdoc/>
public string StdOut { get; private set; }
/// <inheritdoc/>
public string StdErr { get; private set; }
/// <inheritdoc/>
public CommandExecutionDisposition Disposition { get; private set; }
internal static RemoteProcessResult CreateForLocalRun() => new() { ShouldRunLocally = true };
internal static RemoteProcessResult CreateFromCompletedProcess(int exitCode, string stdOut, string stdErr, CommandExecutionDisposition disposition) =>
new() { ExitCode = exitCode, StdOut = stdOut, StdErr = stdErr, Disposition = disposition };
}
}

29
Public/Src/Engine/UnitTests/ExternalToolTests/TestRemoteProcessFactory.cs Normal file
Просмотреть файл

@ -0,0 +1,29 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Processes.Remoting;
namespace ExternalToolTest.BuildXL.Scheduler
{
/// <summary>
/// Remote process factory for testing.
/// </summary>
internal class TestRemoteProcessFactory : IRemoteProcessFactory
{
private readonly bool m_shouldRunLocally;
/// <summary>
/// Creates an instance of <see cref="TestRemoteProcess"/>.
/// </summary>
public TestRemoteProcessFactory(bool shouldRunLocally = false) => m_shouldRunLocally = shouldRunLocally;
public async Task<IRemoteProcess> CreateAndStartAsync(RemoteCommandExecutionInfo remoteCommandInfo, CancellationToken cancellationToken)
{
var rp = new TestRemoteProcess(remoteCommandInfo, cancellationToken, m_shouldRunLocally);
await rp.StartAsync();
return rp;
}
}
}

51
Public/Src/Engine/UnitTests/Processes/RetryInfoTests.cs Normal file
Просмотреть файл

@ -0,0 +1,51 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.IO;
using BuildXL.Processes;
using Test.BuildXL.TestUtilities.Xunit;
using Xunit;
using Xunit.Abstractions;
namespace Test.BuildXL.Processes
{
public sealed class RetryInfoTests : XunitBuildXLTest
{
private ITestOutputHelper TestOutput { get; }
public RetryInfoTests(ITestOutputHelper output)
: base(output) => TestOutput = output;
[Theory]
[InlineData(RetryReason.ResourceExhaustion, false, true)]
[InlineData(RetryReason.UserSpecifiedExitCode, true, false)]
[InlineData(RetryReason.VmExecutionError, true, true)]
[InlineData(RetryReason.RemoteFallback, false, true)]
public void BasicTest(RetryReason reason, bool canBeRetriedInline, bool canBeRetriedByReschedule)
{
RetryInfo retryInfo = RetryInfo.GetDefault(reason);
Verify(retryInfo);
using var ms = new MemoryStream();
retryInfo.Serialize(new BinaryWriter(ms));
ms.Seek(0, SeekOrigin.Begin);
retryInfo = RetryInfo.Deserialize(new BinaryReader(ms));
Verify(retryInfo);
void Verify(RetryInfo ri)
{
if (canBeRetriedInline)
{
XAssert.IsTrue(ri.CanBeRetriedInline());
}
if (canBeRetriedByReschedule)
{
XAssert.IsTrue(ri.CanBeRetriedByReschedule());
}
}
}
}
}

2
Public/Src/Tools/SandboxedProcessExecutor/ConsoleLogger.cs
Просмотреть файл

@ -41,6 +41,8 @@ namespace BuildXL.SandboxedProcessExecutor
lock (m_lock)
{
ConsoleColor original = Console.ForegroundColor;
string dateTime = DateTime.UtcNow.ToString("yyyy.MM.dd HH:mm:ss.fff");
message = $"{dateTime} [{type.ToString().ToUpperInvariant()}]: {message}";
switch (type)
{

30
Public/Src/Tools/SandboxedProcessExecutor/Executor.cs
Просмотреть файл

@ -7,6 +7,7 @@ using System.Diagnostics;
using System.Diagnostics.ContractsLight;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Native.IO;
@ -183,6 +184,9 @@ namespace BuildXL.SandboxedProcessExecutor
if (executeResult.result != null)
{
PostProcessSandboxedProcessResult(sandboxedProcessInfo, executeResult.result);
LogSummary(executeResult.result);
if (m_configuration.PrintObservedAccesses)
{
PrintObservedAccesses(sandboxedProcessInfo.PathTable, executeResult.result);
@ -197,6 +201,32 @@ namespace BuildXL.SandboxedProcessExecutor
return executeResult.exitCode;
}
private void LogSummary(SandboxedProcessResult result)
{
m_logger.LogInfo($"Process exited with exit code '{result.ExitCode}' in {result.PrimaryProcessTimes.TotalWallClockTime.TotalMilliseconds} ms");
}
private void PostProcessSandboxedProcessResult(SandboxedProcessInfo info, SandboxedProcessResult result)
{
m_logger.LogInfo("Post processing sandboxed process result");
if (result.FileAccesses != null)
{
if (info.RemoteSandboxedProcessData != null)
{
// TODO: Hack! Hack!
// This changes is done so that AnyBuild does not try to send untracked files as inputs/outputs.
// This changes the file accesses that BuildXL will see.
// Ideally, this filtration should be done in AnyBuild when processing the result of SandboxedProcessResult
// coming from this executor.
HashSet<ReportedFileAccess> trackedAccesses = result
.FileAccesses
.Where(fa => !info.RemoteSandboxedProcessData.IsUntracked(fa.GetPath(info.PathTable))).ToHashSet();
result.FileAccesses = trackedAccesses;
}
}
}
private void PrintObservedAccesses(PathTable pathTable, SandboxedProcessResult result)
{
var accesses = new List<ReportedFileAccess>();

5
Public/Src/Utilities/Configuration/AdminRequiredProcessExecutionMode.cs
Просмотреть файл

@ -4,7 +4,7 @@
namespace BuildXL.Utilities.Configuration
{
/// <summary>
/// The verbosity level for logging
/// Execution mode for processes that require admin privilege.
/// </summary>
public enum AdminRequiredProcessExecutionMode : byte
{
@ -17,7 +17,8 @@ namespace BuildXL.Utilities.Configuration
/// The admin-required sandboxed process will be launched from a separate sandboxed process executor tool.
/// </summary>
/// <remarks>
/// This mode is mainly used for testing purpose.
/// This mode is introduced to check the functionality of process execution via sandboxed process executor tool, which "simulates"
/// the process execution in VM. This mode is mainly used for testing purpose.
/// </remarks>
ExternalTool,

5
Public/Src/Utilities/Configuration/ISandboxConfiguration.cs
Просмотреть файл

@ -272,11 +272,6 @@ namespace BuildXL.Utilities.Configuration
/// </summary>
int VmConcurrencyLimit { get; }
/// <summary>
/// Whether or not to remote all process pips.
/// </summary>
bool RemoteAllProcesses { get; }
/// <summary>
/// List of directory paths where full reparse point resolving will be applied to any path under them.
/// This list is only considered when <see cref="IUnsafeSandboxConfiguration.IgnoreFullReparsePointResolving"/> is set to true and<see cref="IUnsafeSandboxConfiguration.EnableFullReparsePointResolving"/> is set to false.

73
Public/Src/Utilities/Configuration/IScheduleConfiguration.cs
Просмотреть файл

@ -52,8 +52,12 @@ namespace BuildXL.Utilities.Configuration
int MaxChooseWorkerCacheLookup { get; }
/// <summary>
/// Specifies the maximum number of processes that BuildXL will launch at one time. The default value is 25% more than the total number of processors in the current machine.
/// Specifies the maximum number of processes that BuildXL will launch at one time locally. The default value is the total number of processors in the current machine.
/// </summary>
/// <remarks>
/// This configuration is used also to control parallelism of components other than the number or processes to execute. For proper control of maximum number of processes,
/// one should use <see cref="EffectiveMaxProcesses"/>.
/// </remarks>
int MaxProcesses { get; }
/// <summary>
@ -424,5 +428,72 @@ namespace BuildXL.Utilities.Configuration
/// Updates file content table by scanning change journal.
/// </summary>
bool UpdateFileContentTableByScanningChangeJournal { get; }
/// <summary>
/// Makes scheduler aware of process remoting via AnyBuild.
/// </summary>
bool EnableProcessRemoting { get; }
/// <summary>
/// Number of remote leases for process remoting.
/// </summary>
/// <remarks>
/// This setting determines the maximum number of processes that can be executed remotely at one time
/// when <see cref="EnableProcessRemoting"/> is true.
///
/// AnyBuild agents have a leasing mechanism for executing processes in them. AnyBuild client needs to get a lease
/// from an agent in order to execute a process in that agent. The more agents we have, the more leases available, and
/// the more processes can be executed remotely.
///
/// In the current implementation, BuildXL does not know the number of available leases (or agents). So, in conjuction with <see cref="MaxProcesses"/>,
/// this number is only used to increase the number of ready-to-execute processes that can be released at one time by the CPU dispatcher queue. Having
/// more released processes gives the scheduler chance to execute some of them remotely.
///
/// TODO: Get information about agent leases from AnyBuild, and so this number can be set dynamically.
/// </remarks>
int? NumOfRemoteAgentLeases { get; }
/// <summary>
/// Tags indicating that the process can run on remote agent.
/// </summary>
/// <remarks>
/// These tags are only applicable when <see cref="EnableProcessRemoting"/> is true.
/// </remarks>
IReadOnlyList<string> ProcessCanRunRemoteTags { get; }
/// <summary>
/// Tags indicating that the process must run locally.
/// </summary>
/// <remarks>
/// These tags are only applicable when <see cref="EnableProcessRemoting"/> is true.
/// </remarks>
IReadOnlyList<string> ProcessMustRunLocalTags { get; }
/// <summary>
/// Specifies the maximum number of processes that BuildXL will execute (or launch) at one time.
/// </summary>
/// <remarks>
/// This setting includes the maximum number of processes that BuildXL will execute (or launch) locally at one time, as
/// specified by <see cref="MaxProcesses"/>. This setting also includes the maximum number of processes that BuildXL
/// will execute in remote agents when <see cref="EnableProcessRemoting"/> is true. Thus, this configuration should not
/// be less than <see cref="MaxProcesses"/>.
///
/// The reason for introducing this configuration is because <see cref="MaxProcesses"/> is also used to control parallelism of
/// components other than the number of processes to execute.
///
/// By default the value is the sum of <see cref="MaxProcesses"/> and <see cref="NumOfRemoteAgentLeases"/>. If <see cref="EnableProcessRemoting"/>
/// is false, then <see cref="NumOfRemoteAgentLeases"/> is assumed to be 0.
///
/// Note that, even though we have <see cref="MaxProcesses"/> + <see cref="NumOfRemoteAgentLeases"/> processes ready to execute, that does not mean
/// that <see cref="MaxProcesses"/> will execute locally and <see cref="NumOfRemoteAgentLeases"/> will execute remotely. The scheduler may decide
/// to execute <see cref="MaxProcesses"/> locally, but only start remoting processes when certain threshold is exceeded. However, the number
/// of (ready-to-execute) processes that will be released by the (CPU) dispatcher queue will not exceed <see cref="EffectiveMaxProcesses"/>.
/// </remarks>
int EffectiveMaxProcesses { get; }
/// <summary>
/// The multiplier for <see cref="MaxProcesses"/> that determines when the scheduler starts to execute process pips remotely when <see cref="EnableProcessRemoting"/> is true.
/// </summary>
double RemotingThresholdMultiplier { get; }
}
}

5
Public/Src/Utilities/Configuration/Mutable/SandboxConfiguration.cs
Просмотреть файл

@ -52,7 +52,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
PreserveOutputsForIncrementalTool = false;
GlobalUnsafePassthroughEnvironmentVariables = new List<string>();
VmConcurrencyLimit = 0;
RemoteAllProcesses = false;
DirectoriesToEnableFullReparsePointParsing = new List<AbsolutePath>();
}
@ -103,7 +102,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
PreserveOutputsForIncrementalTool = template.PreserveOutputsForIncrementalTool;
GlobalUnsafePassthroughEnvironmentVariables = new List<string>(template.GlobalUnsafePassthroughEnvironmentVariables);
VmConcurrencyLimit = template.VmConcurrencyLimit;
RemoteAllProcesses = template.RemoteAllProcesses;
DirectoriesToEnableFullReparsePointParsing = pathRemapper.Remap(template.DirectoriesToEnableFullReparsePointParsing);
}
@ -267,9 +265,6 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
public int VmConcurrencyLimit { get; set; }
/// <inheritdoc />
public bool RemoteAllProcesses { get; set; }
/// <nodoc />
public List<AbsolutePath> DirectoriesToEnableFullReparsePointParsing { get; set; }

38
Public/Src/Utilities/Configuration/Mutable/ScheduleConfiguration.cs
Просмотреть файл

@ -86,6 +86,11 @@ namespace BuildXL.Utilities.Configuration.Mutable
PluginLocations = new List<AbsolutePath>();
TreatAbsentDirectoryAsExistentUnderOpaque = true;
EnableProcessRemoting = false;
ProcessCanRunRemoteTags = new List<string>();
ProcessMustRunLocalTags = new List<string>();
RemotingThresholdMultiplier = 1.5;
}
/// <nodoc />
@ -169,6 +174,13 @@ namespace BuildXL.Utilities.Configuration.Mutable
TreatAbsentDirectoryAsExistentUnderOpaque = template.TreatAbsentDirectoryAsExistentUnderOpaque;
MaxWorkersPerModule = template.MaxWorkersPerModule;
UseHistoricalCpuUsageInfo = template.UseHistoricalCpuUsageInfo;
EnableProcessRemoting = template.EnableProcessRemoting;
NumOfRemoteAgentLeases = template.NumOfRemoteAgentLeases;
ProcessCanRunRemoteTags = new List<string>(template.ProcessCanRunRemoteTags);
ProcessMustRunLocalTags = new List<string>(template.ProcessMustRunLocalTags);
RemotingThresholdMultiplier = template.RemotingThresholdMultiplier;
StopDirtyOnSucceedFastPips = template.StopDirtyOnSucceedFastPips;
}
@ -420,5 +432,31 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <inheritdoc />
public bool UpdateFileContentTableByScanningChangeJournal { get; set; }
/// <inheritdoc />
public bool EnableProcessRemoting { get; set; }
/// <inheritdoc />
public int? NumOfRemoteAgentLeases { get; set; }
/// <inheritdoc />
public List<string> ProcessCanRunRemoteTags { get; set; }
/// <inheritdoc />
IReadOnlyList<string> IScheduleConfiguration.ProcessCanRunRemoteTags => ProcessCanRunRemoteTags;
/// <inheritdoc />
public List<string> ProcessMustRunLocalTags { get; set; }
/// <inheritdoc />
IReadOnlyList<string> IScheduleConfiguration.ProcessMustRunLocalTags => ProcessMustRunLocalTags;
/// <inheritdoc />
public int EffectiveMaxProcesses => MaxProcesses + (EnableProcessRemoting ? (NumOfRemoteAgentLeasesValue < 0 ? 0 : NumOfRemoteAgentLeasesValue) : 0);
/// <inheritdoc />
public double RemotingThresholdMultiplier { get; set; }
private int NumOfRemoteAgentLeasesValue => NumOfRemoteAgentLeases ?? 2 * MaxProcesses;
}
}

2
Public/Src/Utilities/UnitTests/Configuration/ConstructorTests.cs
Просмотреть файл

@ -65,7 +65,7 @@ namespace Test.BuildXL.Utilities
foreach (var property in type.GetTypeInfo().GetProperties())
{
if (!property.GetGetMethod().IsStatic)
if (!property.GetGetMethod().IsStatic && property.GetSetMethod() != null)
{
var newValue = CreateInstance(context, property.PropertyType, booleanDefault);
property.SetValue(instance, newValue);