microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

DropDaemon refactoring (#528) 

AB#1552016
This commit is contained in:
olkononenko 2019-07-18 01:06:46 -07:00 коммит произвёл GitHub
Родитель f6e1cab039
Коммит 5d2cfeb8b3
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
32 изменённых файлов: 1762 добавлений и 1563 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
Private/DistributedIntegrationTest/BuildLocalDev.cmd
Просмотреть файл

@ -8,7 +8,7 @@ if NOT DEFINED ENLISTMENTROOT (
echo =======================================================
echo Building BuildXL
echo =======================================================
call %ENLISTMENTROOT%\bxl.cmd -deploy dev /server- /f:output='Out\Bin\debug\net472\*'
call %ENLISTMENTROOT%\bxl.cmd -deploydev /server-
if %ERRORLEVEL% NEQ 0 (
echo.
echo ERROR: BuildXL build failed.
@ -16,7 +16,7 @@ if %ERRORLEVEL% NEQ 0 (
endlocal && exit /b 1
)
set BUILDXL_BIN_DIRECTORY=%ENLISTMENTROOT%\Out\Bin\Debug\net472
set BUILDXL_BIN_DIRECTORY=%ENLISTMENTROOT%\Out\Bin\Debug\win-x64
if NOT DEFINED TF_ROLLING_DROPNAME (
set TF_ROLLING_DROPNAME=%USERNAME%-%random%

8
Public/Sdk/SelfHost/BuildXL/BuildXLSdk.dsc
Просмотреть файл

@ -115,11 +115,17 @@ export const restrictTestRunToDebugNet461OnWindows =
(qualifier.targetFramework !== "netcoreapp3.0" && qualifier.targetFramework !== "net472") ||
(Context.isWindowsOS() && qualifier.targetRuntime === "osx-x64");
@@public
/***
* Whether service pip daemon tooling is included with the BuildXL deployment
*/
export const isDaemonToolingEnabled = Flags.isMicrosoftInternal && isFullFramework;
/***
* Whether drop tooling is included with the BuildXL deployment
*/
@@public
export const isDropToolingEnabled = Flags.isMicrosoftInternal && isFullFramework;
export const isDropToolingEnabled = isDaemonToolingEnabled && Flags.isMicrosoftInternal && isFullFramework;
namespace Flags {
export declare const qualifier: {};

131
Public/Src/Tools/DropDaemon/Command.cs
Просмотреть файл

@ -1,131 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.Interfaces;
using BuildXL.Utilities.CLI;
namespace Tool.DropDaemon
{
internal delegate int ClientAction(ConfiguredCommand conf, IClient rpc);
internal delegate Task<IIpcResult> ServerAction(ConfiguredCommand conf, Daemon daemon);
/// <summary>
/// A command has a name, description, a list of options it supports, and two actions:
/// one for executing this command on the client, and one for executing it on the server.
///
/// When this program (DropDaemon.exe) is invoked, command line arguments are parsed to
/// determine the command specified by the user. That command is then interpreted by
/// executing its <see cref="ClientAction"/>. Most of DropDaemon's commands will be
/// RPC calls, i.e., when a command is received via the command line, it is to be
/// marshaled and sent over to a running DropDaemon server via an RPC. In such a case,
/// the client action simply invokes <see cref="IClient.Send"/>. When an RPC is
/// received by a DropDaemon server (<see cref="Daemon.ParseAndExecuteCommand"/>), a
/// <see cref="Command"/> is unmarshaled from the payload of the RPC operation and the
/// command is interpreted on the server by executing its <see cref="ServerAction"/>.
/// </summary>
/// <remarks>
/// Immutable.
/// </remarks>
internal sealed class Command
{
/// <summary>A unique command name.</summary>
internal string Name { get; }
/// <summary>Arbitrary description.</summary>
internal string Description { get; }
/// <summary>Options that may/must be passed to this command.</summary>
internal IReadOnlyCollection<Option> Options { get; }
/// <summary>Action to be executed when this command is received via the command line.</summary>
internal ClientAction ClientAction { get; }
/// <summary>Action to be executed when this command is received via an RPC call.</summary>
internal ServerAction ServerAction { get; }
/// <summary>Whether this command requires an IpcClient; defaults to true.</summary>
internal bool NeedsIpcClient { get; }
/// <nodoc />
internal Command(
string name,
IEnumerable<Option> options = null,
ServerAction serverAction = null,
ClientAction clientAction = null,
string description = null,
bool needsIpcClient = true)
{
Contract.Requires(name != null);
Name = name;
ServerAction = serverAction;
ClientAction = clientAction;
Description = description;
Options = options.ToList();
NeedsIpcClient = needsIpcClient;
}
/// <summary>
/// Performs a functional composition of a number of <see cref="ServerAction"/> functions,
/// where the results are merged by calling <see cref="IpcResult.Merge(IIpcResult, IIpcResult)"/>.
/// </summary>
internal static ServerAction Compose(params ServerAction[] actions)
{
Contract.Requires(actions != null);
Contract.Requires(actions.Length > 0);
var first = actions.First();
return actions.Skip(1).Aggregate(first, (accumulator, currentAction) => new ServerAction(async (conf, daemon) =>
{
var lhsResult = await accumulator(conf, daemon);
var rhsResult = await currentAction(conf, daemon);
return IpcResult.Merge(lhsResult, rhsResult);
}));
}
internal string Usage(IParser parser)
{
var result = new StringBuilder();
var tab = " ";
result.AppendLine("NAME");
result.Append(tab).Append(Name).Append(" - ").AppendLine(Description);
result.AppendLine();
result.AppendLine("SWITCHES");
var optsSorted = Options
.OrderBy(o => o.IsRequired ? 0 : 1)
.ThenBy(o => o.LongName)
.ToArray();
result.AppendLine(parser.Usage(optsSorted, tab, tab));
return result.ToString();
}
}
/// <summary>
/// Simple wrapper class that holds a <see cref="Command"/> and a <see cref="Config"/>
/// containing actual values for the command's <see cref="Command.Options"/>.
/// </summary>
internal sealed class ConfiguredCommand
{
internal Command Command { get; }
internal Config Config { get; }
internal ILogger Logger { get; }
internal ConfiguredCommand(Command command, Config config, ILogger logger)
{
Command = command;
Config = config;
Logger = logger;
}
internal T Get<T>(Option<T> option) => option.GetValue(Config);
}
}

438
Public/Src/Tools/DropDaemon/Daemon.cs
Просмотреть файл

@ -1,438 +0,0 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Diagnostics.ContractsLight;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using BuildXL.Ipc;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.ExternalApi;
using BuildXL.Ipc.Interfaces;
using BuildXL.Tracing.CloudBuild;
using BuildXL.Utilities.CLI;
using BuildXL.Utilities.Tracing;
using JetBrains.Annotations;
using Microsoft.VisualStudio.Services.Common;
using Microsoft.VisualStudio.Services.Drop.WebApi;
using Newtonsoft.Json;
using static Tool.DropDaemon.Statics;
namespace Tool.DropDaemon
{
/// <summary>
/// Responsible for accepting and handling TCP/IP connections from clients.
/// </summary>
public sealed class Daemon : IDisposable, IIpcOperationExecutor
{
/// <summary>Prefix for the error message of the exception that gets thrown when a symlink is attempted to be added to drop.</summary>
internal const string SymlinkAddErrorMessagePrefix = "SymLinks may not be added to drop: ";
private const string LogFileName = "DropDaemon";
/// <nodoc/>
public const string DropDLogPrefix = "(DropD) ";
/// <summary>Daemon configuration.</summary>
public DaemonConfig Config { get; }
/// <summary>Drop configuration.</summary>
public DropConfig DropConfig { get; }
/// <summary>Task to wait on for the completion result.</summary>
public Task Completion => m_server.Completion;
/// <summary>Name of the drop this daemon is constructing.</summary>
public string DropName => DropConfig.Name;
/// <summary>Client for talking to BuildXL.</summary>
[CanBeNull]
public Client ApiClient { get; }
private readonly Task<IDropClient> m_dropClientTask;
private readonly ICloudBuildLogger m_etwLogger;
private readonly IServer m_server;
private readonly IParser m_parser;
private readonly ILogger m_logger;
/// <nodoc />
public ILogger Logger => m_logger;
/// <nodoc />
public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task<IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Contract.Requires(dropConfig != null);
Config = daemonConfig;
DropConfig = dropConfig;
m_parser = parser;
ApiClient = client;
m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient)new VsoClient(m_logger, dropConfig));
}
/// <summary>
/// Starts to listen for client connections. As soon as a connection is received,
/// it is placed in an action block from which it is picked up and handled asynchronously
/// (in the <see cref="ParseAndExecuteCommand"/> method).
/// </summary>
public void Start()
{
m_server.Start(this);
}
/// <summary>
/// Requests shut down, causing this daemon to immediatelly stop listening for TCP/IP
/// connections. Any pending requests, however, will be processed to completion.
/// </summary>
public void RequestStop()
{
m_server.RequestStop();
}
/// <summary>
/// Calls <see cref="RequestStop"/> then waits for <see cref="Completion"/>.
/// </summary>
public Task RequestStopAndWaitForCompletionAsync()
{
RequestStop();
return Completion;
}
/// <summary>
/// Synchronous version of <see cref="CreateAsync"/>
/// </summary>
public IIpcResult Create()
{
return CreateAsync().GetAwaiter().GetResult();
}
/// <summary>
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task<IIpcResult> CreateAsync()
{
DropCreationEvent dropCreationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
return dropCreationEvent.Succeeded
? IpcResult.Success(Inv("Drop {0} created.", DropName))
: new IpcResult(IpcResultStatus.ExecutionError, dropCreationEvent.ErrorMessage);
}
/// <summary>
/// Invokes the 'drop addfile' operation by delegating to <see cref="IDropClient.AddFileAsync"/>.
/// Handles drop-related exceptions by omitting their stack traces.
/// </summary>
public Task<IIpcResult> AddFileAsync(IDropItem dropItem)
{
return AddFileAsync(dropItem, IsSymLinkOrMountPoint);
}
internal async Task<IIpcResult> AddFileAsync(IDropItem dropItem, Func<string, bool> symlinkTester)
{
Contract.Requires(dropItem != null);
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
if (File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
{
return new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath);
}
return await WrapDropErrorsIntoIpcResult(async () =>
{
IDropClient dropClient = await m_dropClientTask;
AddFileResult result = await dropClient.AddFileAsync(dropItem);
return IpcResult.Success(Inv(
"File '{0}' {1} under '{2}' in drop '{3}'.",
dropItem.FullFilePath,
result,
dropItem.RelativeDropPath,
DropName));
});
}
/// <summary>
/// Gets file's path relative to a given root.
/// The method assumes that file is under the root; however it does not enforce this assumption.
/// </summary>
private string GetRelativePath(string root, string file)
{
var rootEndsWithSlash =
root[root.Length - 1] == Path.DirectorySeparatorChar
|| root[root.Length - 1] == Path.AltDirectorySeparatorChar;
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
}
/// <summary>
/// Synchronous version of <see cref="FinalizeAsync"/>
/// </summary>
public IIpcResult Finalize()
{
return FinalizeAsync().GetAwaiter().GetResult();
}
/// <summary>
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task<IIpcResult> FinalizeAsync()
{
var dropFinalizationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
return dropFinalizationEvent.Succeeded
? IpcResult.Success(Inv("Drop {0} finalized", DropName))
: new IpcResult(IpcResultStatus.ExecutionError, dropFinalizationEvent.ErrorMessage);
}
/// <inheritdoc />
public void Dispose()
{
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
{
ReportStatisticsAsync().GetAwaiter().GetResult();
m_dropClientTask.Result.Dispose();
}
m_server.Dispose();
ApiClient?.Dispose();
m_logger.Dispose();
}
/// <summary>
/// Invokes the 'drop create' operation by delegating to <see cref="IDropClient.CreateAsync"/>.
///
/// If successful, returns <see cref="DropCreationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
/// set to true, <see cref="DropCreationEvent.DropExpirationInDays"/> set to drop expiration in days,
/// and <see cref="DropOperationBaseEvent.AdditionalInformation"/> set to the textual representation
/// of the returned <see cref="DropItem"/> object.
///
/// Doesn't handle any exceptions.
/// </summary>
private async Task<DropCreationEvent> InternalCreateAsync()
{
IDropClient dropClient = await m_dropClientTask;
DropItem dropItem = await dropClient.CreateAsync();
return new DropCreationEvent()
{
Succeeded = true,
AdditionalInformation = DropItemToString(dropItem),
DropExpirationInDays = ComputeDropItemExpiration(dropItem),
};
}
/// <summary>
/// Invokes the 'drop finalize' operation by delegating to <see cref="IDropClient.FinalizeAsync"/>.
///
/// If successful, returns <see cref="DropFinalizationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
/// set to true.
///
/// Doesn't handle any exceptions.
/// </summary>
private async Task<DropFinalizationEvent> InternalFinalizeAsync()
{
IDropClient dropClient = await m_dropClientTask;
await dropClient.FinalizeAsync();
return new DropFinalizationEvent()
{
Succeeded = true,
};
}
private async Task ReportStatisticsAsync()
{
IDropClient dropClient = await m_dropClientTask;
var stats = dropClient.GetStats();
if (stats != null && stats.Any())
{
// log stats
m_logger.Info("Statistics: ");
m_logger.Info(string.Join(Environment.NewLine, stats.Select(s => s.Key + " = " + s.Value)));
stats.Add(nameof(DropConfig) + nameof(DropConfig.MaxParallelUploads), DropConfig.MaxParallelUploads);
stats.Add(nameof(DropConfig) + nameof(DropConfig.NagleTime), (long)DropConfig.NagleTime.TotalMilliseconds);
stats.Add(nameof(DropConfig) + nameof(DropConfig.BatchSize), DropConfig.BatchSize);
stats.Add(nameof(DropConfig) + nameof(DropConfig.EnableChunkDedup), DropConfig.EnableChunkDedup ? 1 : 0);
stats.Add("DaemonConfig" + nameof(Config.MaxConcurrentClients), Config.MaxConcurrentClients);
stats.Add("DaemonConfig" + nameof(Config.ConnectRetryDelay), (long)Config.ConnectRetryDelay.TotalMilliseconds);
stats.Add("DaemonConfig" + nameof(Config.MaxConnectRetries), Config.MaxConnectRetries);
stats.AddRange(m_counters.AsStatistics());
// report stats to BuildXL (if m_client is specified)
if (ApiClient != null)
{
var possiblyReported = await ApiClient.ReportStatistics(stats);
if (possiblyReported.Succeeded && possiblyReported.Result)
{
m_logger.Info("Statistics successfully reported to BuildXL.");
}
else
{
var errorDescription = possiblyReported.Succeeded ? string.Empty : possiblyReported.Failure.Describe();
m_logger.Warning("Reporting stats to BuildXL failed. " + errorDescription);
}
}
}
else
{
m_logger.Info("No stats recorded by drop client of type " + dropClient.GetType().Name);
}
}
private static Task<IIpcResult> WrapDropErrorsIntoIpcResult(Func<Task<IIpcResult>> factory)
{
return HandleKnownErrors(
factory,
(errorMessage) => new IpcResult(IpcResultStatus.ExecutionError, errorMessage));
}
private static Task<TDropEvent> WrapDropErrorsIntoDropEtwEvent<TDropEvent>(Func<Task<TDropEvent>> factory) where TDropEvent : DropOperationBaseEvent
{
return HandleKnownErrors(
factory,
(errorMessage) =>
{
var dropEvent = Activator.CreateInstance<TDropEvent>();
dropEvent.Succeeded = false;
dropEvent.ErrorMessage = errorMessage;
return dropEvent;
});
}
private static async Task<TResult> HandleKnownErrors<TResult>(Func<Task<TResult>> factory, Func<string, TResult> errorValueFactory)
{
try
{
return await factory();
}
catch (VssUnauthorizedException e)
{
return errorValueFactory("[DROP AUTH ERROR] " + e.Message);
}
catch (DropServiceException e)
{
return errorValueFactory("[DROP SERVICE ERROR] " + e.Message);
}
catch (DropDaemonException e)
{
return errorValueFactory("[DROP DAEMON ERROR] " + e.Message);
}
}
private static string DropItemToString(DropItem dropItem)
{
try
{
return dropItem?.ToJson().ToString();
}
#pragma warning disable ERP022 // TODO: This should really handle specific errors
catch
{
return null;
}
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
}
private static int ComputeDropItemExpiration(DropItem dropItem)
{
DateTime? expirationDate;
return dropItem.TryGetExpirationTime(out expirationDate) || expirationDate.HasValue
? (int)expirationDate.Value.Subtract(DateTime.UtcNow).TotalDays
: -1;
}
private async Task<T> SendDropEtwEvent<T>(Task<T> task) where T : DropOperationBaseEvent
{
long startTime = DateTime.UtcNow.Ticks;
T dropEvent = null;
try
{
dropEvent = await task;
return dropEvent;
}
finally
{
// if 'task' failed, create an event indicating an error
if (dropEvent == null)
{
dropEvent = Activator.CreateInstance<T>();
dropEvent.Succeeded = false;
dropEvent.ErrorMessage = "internal error";
}
// common properties: execution time, drop type, drop url
dropEvent.ElapsedTimeTicks = DateTime.UtcNow.Ticks - startTime;
dropEvent.DropType = "VsoDrop";
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
{
dropEvent.DropUrl = (await m_dropClientTask).DropUrl;
}
// send event
m_etwLogger.Log(dropEvent);
}
}
private readonly CounterCollection<DaemonCounter> m_counters = new CounterCollection<DaemonCounter>();
private enum DaemonCounter
{
/// <nodoc/>
[CounterType(CounterType.Stopwatch)]
ParseArgsDuration,
/// <nodoc/>
[CounterType(CounterType.Stopwatch)]
ServerActionDuration,
/// <nodoc/>
QueueDurationMs,
}
private async Task<IIpcResult> ParseAndExecuteCommand(IIpcOperation operation)
{
string cmdLine = operation.Payload;
m_logger.Verbose("Command received: {0}", cmdLine);
ConfiguredCommand conf;
using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration))
{
conf = Program.ParseArgs(cmdLine, m_parser);
}
IIpcResult result;
using (m_counters.StartStopwatch(DaemonCounter.ServerActionDuration))
{
result = await conf.Command.ServerAction(conf, this);
}
TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime;
m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds);
return result;
}
Task<IIpcResult> IIpcOperationExecutor.ExecuteAsync(IIpcOperation operation)
{
Contract.Requires(operation != null);
return ParseAndExecuteCommand(operation);
}
}
}

914
Public/Src/Tools/DropDaemon/DropDaemon.cs Normal file
Просмотреть файл

@ -0,0 +1,914 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.ExternalApi;
using BuildXL.Ipc.Interfaces;
using BuildXL.Scheduler;
using BuildXL.Storage;
using BuildXL.Tracing.CloudBuild;
using BuildXL.Utilities;
using BuildXL.Utilities.CLI;
using BuildXL.Utilities.Tasks;
using Microsoft.VisualStudio.Services.Common;
using Microsoft.VisualStudio.Services.Drop.WebApi;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Tool.ServicePipDaemon;
using static BuildXL.Utilities.FormattableStringEx;
using static Tool.ServicePipDaemon.Statics;
namespace Tool.DropDaemon
{
/// <summary>
/// Responsible for accepting and handling TCP/IP connections from clients.
/// </summary>
public sealed class DropDaemon : ServicePipDaemon.ServicePipDaemon, IDisposable, IIpcOperationExecutor
{
private const int ServicePointParallelismForDrop = 200;
private const string IncludeAllFilter = ".*";
/// <summary>Prefix for the error message of the exception that gets thrown when a symlink is attempted to be added to drop.</summary>
internal const string SymlinkAddErrorMessagePrefix = "SymLinks may not be added to drop: ";
private const string LogFileName = "DropDaemon";
/// <nodoc/>
public const string DropDLogPrefix = "(DropD) ";
private static readonly int s_minIoThreadsForDrop = Environment.ProcessorCount * 10;
private static readonly int s_minWorkerThreadsForDrop = Environment.ProcessorCount * 10;
internal static readonly List<Option> DropConfigOptions = new List<Option>();
/// <summary>Drop configuration.</summary>
public DropConfig DropConfig { get; }
/// <summary>Name of the drop this daemon is constructing.</summary>
public string DropName => DropConfig.Name;
private readonly Task<IDropClient> m_dropClientTask;
internal static IEnumerable<Command> SupportedCommands => Commands.Values;
#region Options and commands
internal static readonly StrOption DropServiceConfigFile = RegisterDaemonConfigOption(new StrOption("dropServiceConfigFile")
{
ShortName = "c",
HelpText = "Drop service configuration file",
DefaultValue = null,
Expander = (fileName) =>
{
var json = System.IO.File.ReadAllText(fileName);
var jObject = JObject.Parse(json);
return jObject.Properties().Select(prop => new ParsedOption(PrefixKind.Long, prop.Name, prop.Value.ToString()));
},
});
internal static readonly StrOption DropNameOption = RegisterDropConfigOption(new StrOption("name")
{
ShortName = "n",
HelpText = "Drop name",
IsRequired = true,
});
internal static readonly UriOption DropEndpoint = RegisterDropConfigOption(new UriOption("service")
{
ShortName = "s",
HelpText = "Drop endpoint URI",
IsRequired = true,
});
internal static readonly IntOption BatchSize = RegisterDropConfigOption(new IntOption("batchSize")
{
ShortName = "bs",
HelpText = "OBSOLETE due to the hardcoded config. (Size of batches in which to send 'associate' requests)",
IsRequired = false,
DefaultValue = DropConfig.DefaultBatchSizeForAssociate,
});
internal static readonly IntOption MaxParallelUploads = RegisterDropConfigOption(new IntOption("maxParallelUploads")
{
ShortName = "mpu",
HelpText = "Maximum number of uploads to issue to drop service in parallel",
IsRequired = false,
DefaultValue = DropConfig.DefaultMaxParallelUploads,
});
internal static readonly IntOption NagleTimeMillis = RegisterDropConfigOption(new IntOption("nagleTimeMillis")
{
ShortName = "nt",
HelpText = "OBSOLETE due to the hardcoded config. (Maximum time in milliseconds to wait before triggering a batch 'associate' request)",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultNagleTimeForAssociate.TotalMilliseconds,
});
internal static readonly IntOption RetentionDays = RegisterDropConfigOption(new IntOption("retentionDays")
{
ShortName = "rt",
HelpText = "Drop retention time in days",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultRetention.TotalDays,
});
internal static readonly IntOption HttpSendTimeoutMillis = RegisterDropConfigOption(new IntOption("httpSendTimeoutMillis")
{
HelpText = "Timeout for http requests",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultHttpSendTimeout.TotalMilliseconds,
});
internal static readonly BoolOption EnableTelemetry = RegisterDropConfigOption(new BoolOption("enableTelemetry")
{
ShortName = "t",
HelpText = "Verbose logging",
IsRequired = false,
DefaultValue = DropConfig.DefaultEnableTelemetry,
});
internal static readonly BoolOption EnableChunkDedup = RegisterDropConfigOption(new BoolOption("enableChunkDedup")
{
ShortName = "cd",
HelpText = "Chunk level dedup",
IsRequired = false,
DefaultValue = DropConfig.DefaultEnableChunkDedup,
});
// ==============================================================================
// 'addfile' and 'addartifacts' parameters
// ==============================================================================
internal static readonly StrOption RelativeDropPath = new StrOption("dropPath")
{
ShortName = "d",
HelpText = "Relative drop path",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption HashOptional = new StrOption("hash")
{
ShortName = "h",
HelpText = "VSO file hash",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption FileId = new StrOption("fileId")
{
ShortName = "fid",
HelpText = "BuildXL file identifier",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption Directory = new StrOption("directory")
{
ShortName = "dir",
HelpText = "Directory path",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption DirectoryId = new StrOption("directoryId")
{
ShortName = "dirid",
HelpText = "BuildXL directory identifier",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption RelativeDirectoryDropPath = new StrOption("directoryDropPath")
{
ShortName = "dird",
HelpText = "Relative drop path for directory",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption DirectoryContentFilter = new StrOption("directoryFilter")
{
ShortName = "dcfilter",
HelpText = "Directory content filter (only files that match the filter will be added to drop).",
DefaultValue = null,
IsRequired = false,
IsMultiValue = true,
};
internal static readonly Command StartNoDropCmd = RegisterCommand(
name: "start-nodrop",
description: @"Starts a server process without a backing VSO drop client (useful for testing/pinging the daemon).",
needsIpcClient: false,
clientAction: (conf, _) =>
{
var dropConfig = new DropConfig(string.Empty, new Uri("file://xyz"));
var daemonConfig = CreateDaemonConfig(conf);
var vsoClientTask = TaskSourceSlim.Create<IDropClient>();
vsoClientTask.SetException(new NotSupportedException());
using (var daemon = new DropDaemon(conf.Config.Parser, daemonConfig, dropConfig, vsoClientTask.Task))
{
daemon.Start();
daemon.Completion.GetAwaiter().GetResult();
return 0;
}
});
internal static readonly Command StartCmd = RegisterCommand(
name: "start",
description: "Starts the server process.",
options: DropConfigOptions.Union(new[] { IpcServerMonikerOptional }),
needsIpcClient: false,
clientAction: (conf, _) =>
{
SetupThreadPoolAndServicePoint();
var dropConfig = CreateDropConfig(conf);
var daemonConf = CreateDaemonConfig(conf);
if (daemonConf.MaxConcurrentClients <= 1)
{
conf.Logger.Error($"Must specify at least 2 '{nameof(DaemonConfig.MaxConcurrentClients)}' when running DropDaemon to avoid deadlock when stopping this daemon from a different client");
return -1;
}
using (var client = CreateClient(conf.Get(IpcServerMonikerOptional), daemonConf))
using (var daemon = new DropDaemon(
parser: conf.Config.Parser,
daemonConfig: daemonConf,
dropConfig: dropConfig,
dropClientTask: null,
client: client))
{
daemon.Start();
daemon.Completion.GetAwaiter().GetResult();
return 0;
}
});
private static void SetupThreadPoolAndServicePoint()
{
int workerThreads, ioThreads;
ThreadPool.GetMinThreads(out workerThreads, out ioThreads);
workerThreads = Math.Max(workerThreads, s_minWorkerThreadsForDrop);
ioThreads = Math.Max(ioThreads, s_minIoThreadsForDrop);
ThreadPool.SetMinThreads(workerThreads, ioThreads);
ServicePointManager.DefaultConnectionLimit = Math.Max(ServicePointParallelismForDrop, ServicePointManager.DefaultConnectionLimit);
}
internal static readonly Command StartDaemonCmd = RegisterCommand(
name: "start-daemon",
description: "Starts the server process in background (as daemon).",
options: DropConfigOptions,
needsIpcClient: false,
clientAction: (conf, _) =>
{
using (var daemon = new Process())
{
bool shellExecute = conf.Get(ShellExecute);
daemon.StartInfo.FileName = AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetEntryAssembly());
daemon.StartInfo.Arguments = "start " + conf.Config.Render();
daemon.StartInfo.LoadUserProfile = false;
daemon.StartInfo.UseShellExecute = shellExecute;
daemon.StartInfo.CreateNoWindow = !shellExecute;
daemon.Start();
}
return 0;
});
internal static readonly Command CreateDropCmd = RegisterCommand(
name: "create",
description: "[RPC] Invokes the 'create' operation.",
options: DropConfigOptions,
clientAction: SyncRPCSend,
serverAction: async (conf, dropDaemon) =>
{
var daemon = dropDaemon as DropDaemon;
daemon.Logger.Info("[CREATE]: Started at " + daemon.DropConfig.Service + "/" + daemon.DropName);
IIpcResult result = await daemon.CreateAsync();
daemon.Logger.Info("[CREATE]: " + result);
return result;
});
internal static readonly Command FinalizeDropCmd = RegisterCommand(
name: "finalize",
description: "[RPC] Invokes the 'finalize' operation.",
clientAction: SyncRPCSend,
serverAction: async (conf, dropDaemon) =>
{
var daemon = dropDaemon as DropDaemon;
daemon.Logger.Info("[FINALIZE] Started at" + daemon.DropConfig.Service + "/" + daemon.DropName);
IIpcResult result = await daemon.FinalizeAsync();
daemon.Logger.Info("[FINALIZE] " + result);
return result;
});
internal static readonly Command FinalizeDropAndStopDaemonCmd = RegisterCommand(
name: "finalize-and-stop",
description: "[RPC] Invokes the 'finalize' operation; then stops the daemon.",
clientAction: SyncRPCSend,
serverAction: Command.Compose(FinalizeDropCmd.ServerAction, StopDaemonCmd.ServerAction));
internal static readonly Command AddFileToDropCmd = RegisterCommand(
name: "addfile",
description: "[RPC] invokes the 'addfile' operation.",
options: new Option[] { File, RelativeDropPath, HashOptional },
clientAction: SyncRPCSend,
serverAction: async (conf, dropDaemon) =>
{
var daemon = dropDaemon as DropDaemon;
daemon.Logger.Verbose("[ADDFILE] Started");
string filePath = conf.Get(File);
string hashValue = conf.Get(HashOptional);
var contentInfo = string.IsNullOrEmpty(hashValue) ? null : (FileContentInfo?)FileContentInfo.Parse(hashValue);
var dropItem = new DropItemForFile(filePath, conf.Get(RelativeDropPath), contentInfo);
IIpcResult result = System.IO.File.Exists(filePath)
? await daemon.AddFileAsync(dropItem)
: new IpcResult(IpcResultStatus.ExecutionError, "file '" + filePath + "' does not exist");
daemon.Logger.Verbose("[ADDFILE] " + result);
return result;
});
internal static readonly Command AddArtifactsToDropCmd = RegisterCommand(
name: "addartifacts",
description: "[RPC] invokes the 'addartifacts' operation.",
options: new Option[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath, Directory, DirectoryId, RelativeDirectoryDropPath, DirectoryContentFilter },
clientAction: SyncRPCSend,
serverAction: async (conf, dropDaemon) =>
{
var daemon = dropDaemon as DropDaemon;
daemon.Logger.Verbose("[ADDARTIFACTS] Started");
var result = await AddArtifactsToDropInternalAsync(conf, daemon);
daemon.Logger.Verbose("[ADDARTIFACTS] " + result);
return result;
});
#endregion
/// <summary>
/// The purpose of this ctor is to force 'predictable' initialization of static fields.
/// </summary>
static DropDaemon()
{
// noop
}
/// <nodoc />
public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task<IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger,
rpcProvider,
client)
{
Contract.Requires(dropConfig != null);
DropConfig = dropConfig;
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient)new VsoClient(m_logger, dropConfig));
}
internal static void EnsureCommandsInitialized()
{
Contract.Assert(Commands != null);
// these operations are quite expensive, however, we expect to call this method only once per drop, so it should cause any perf downgrade
var numCommandsBase = typeof(ServicePipDaemon.ServicePipDaemon).GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(f => f.FieldType == typeof(Command)).Count();
var numCommandsDropD = typeof(DropDaemon).GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(f => f.FieldType == typeof(Command)).Count();
if (Commands.Count != numCommandsBase + numCommandsDropD)
{
Contract.Assert(false, $"The list of commands was not properly initialized (# of initialized commands = {Commands.Count}; # of ServicePipDaemon commands = {numCommandsBase}; # of DropDaemon commands = {numCommandsDropD})");
}
}
/// <summary>
/// Synchronous version of <see cref="CreateAsync"/>
/// </summary>
public IIpcResult Create()
{
return CreateAsync().GetAwaiter().GetResult();
}
/// <summary>
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task<IIpcResult> CreateAsync()
{
DropCreationEvent dropCreationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
return dropCreationEvent.Succeeded
? IpcResult.Success(I($"Drop {DropName} created."))
: new IpcResult(IpcResultStatus.ExecutionError, dropCreationEvent.ErrorMessage);
}
/// <summary>
/// Invokes the 'drop addfile' operation by delegating to <see cref="IDropClient.AddFileAsync"/>.
/// Handles drop-related exceptions by omitting their stack traces.
/// </summary>
public Task<IIpcResult> AddFileAsync(IDropItem dropItem)
{
return AddFileAsync(dropItem, IsSymLinkOrMountPoint);
}
internal async Task<IIpcResult> AddFileAsync(IDropItem dropItem, Func<string, bool> symlinkTester)
{
Contract.Requires(dropItem != null);
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
if (System.IO.File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
{
return new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath);
}
return await WrapDropErrorsIntoIpcResult(async () =>
{
IDropClient dropClient = await m_dropClientTask;
AddFileResult result = await dropClient.AddFileAsync(dropItem);
return IpcResult.Success(I($"File '{dropItem.FullFilePath}' {result} under '{dropItem.RelativeDropPath}' in drop '{DropName}'."));
});
}
/// <summary>
/// Synchronous version of <see cref="FinalizeAsync"/>
/// </summary>
public IIpcResult Finalize()
{
return FinalizeAsync().GetAwaiter().GetResult();
}
/// <summary>
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
/// result of this operation.
/// </summary>
public async Task<IIpcResult> FinalizeAsync()
{
var dropFinalizationEvent =
await SendDropEtwEvent(
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
return dropFinalizationEvent.Succeeded
? IpcResult.Success(I($"Drop {DropName} finalized"))
: new IpcResult(IpcResultStatus.ExecutionError, dropFinalizationEvent.ErrorMessage);
}
/// <inheritdoc />
public new void Dispose()
{
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
{
ReportStatisticsAsync().GetAwaiter().GetResult();
m_dropClientTask.Result.Dispose();
}
base.Dispose();
}
/// <summary>
/// Invokes the 'drop create' operation by delegating to <see cref="IDropClient.CreateAsync"/>.
///
/// If successful, returns <see cref="DropCreationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
/// set to true, <see cref="DropCreationEvent.DropExpirationInDays"/> set to drop expiration in days,
/// and <see cref="DropOperationBaseEvent.AdditionalInformation"/> set to the textual representation
/// of the returned <see cref="DropItem"/> object.
///
/// Doesn't handle any exceptions.
/// </summary>
private async Task<DropCreationEvent> InternalCreateAsync()
{
IDropClient dropClient = await m_dropClientTask;
DropItem dropItem = await dropClient.CreateAsync();
return new DropCreationEvent()
{
Succeeded = true,
AdditionalInformation = DropItemToString(dropItem),
DropExpirationInDays = ComputeDropItemExpiration(dropItem),
};
}
/// <summary>
/// Invokes the 'drop finalize' operation by delegating to <see cref="IDropClient.FinalizeAsync"/>.
///
/// If successful, returns <see cref="DropFinalizationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
/// set to true.
///
/// Doesn't handle any exceptions.
/// </summary>
private async Task<DropFinalizationEvent> InternalFinalizeAsync()
{
IDropClient dropClient = await m_dropClientTask;
await dropClient.FinalizeAsync();
return new DropFinalizationEvent()
{
Succeeded = true,
};
}
private async Task ReportStatisticsAsync()
{
IDropClient dropClient = await m_dropClientTask;
var stats = dropClient.GetStats();
if (stats != null && stats.Any())
{
// log stats
m_logger.Info("Statistics: ");
m_logger.Info(string.Join(Environment.NewLine, stats.Select(s => s.Key + " = " + s.Value)));
stats.Add(nameof(DropConfig) + nameof(DropConfig.MaxParallelUploads), DropConfig.MaxParallelUploads);
stats.Add(nameof(DropConfig) + nameof(DropConfig.NagleTime), (long)DropConfig.NagleTime.TotalMilliseconds);
stats.Add(nameof(DropConfig) + nameof(DropConfig.BatchSize), DropConfig.BatchSize);
stats.Add(nameof(DropConfig) + nameof(DropConfig.EnableChunkDedup), DropConfig.EnableChunkDedup ? 1 : 0);
stats.Add(nameof(DaemonConfig) + nameof(Config.MaxConcurrentClients), Config.MaxConcurrentClients);
stats.Add(nameof(DaemonConfig) + nameof(Config.ConnectRetryDelay), (long)Config.ConnectRetryDelay.TotalMilliseconds);
stats.Add(nameof(DaemonConfig) + nameof(Config.MaxConnectRetries), Config.MaxConnectRetries);
stats.AddRange(m_counters.AsStatistics());
// report stats to BuildXL (if m_client is specified)
if (ApiClient != null)
{
var possiblyReported = await ApiClient.ReportStatistics(stats);
if (possiblyReported.Succeeded && possiblyReported.Result)
{
m_logger.Info("Statistics successfully reported to BuildXL.");
}
else
{
var errorDescription = possiblyReported.Succeeded ? string.Empty : possiblyReported.Failure.Describe();
m_logger.Warning("Reporting stats to BuildXL failed. " + errorDescription);
}
}
}
else
{
m_logger.Info("No stats recorded by drop client of type " + dropClient.GetType().Name);
}
}
private static Task<IIpcResult> WrapDropErrorsIntoIpcResult(Func<Task<IIpcResult>> factory)
{
return HandleKnownErrors(
factory,
(errorMessage) => new IpcResult(IpcResultStatus.ExecutionError, errorMessage));
}
private static Task<TDropEvent> WrapDropErrorsIntoDropEtwEvent<TDropEvent>(Func<Task<TDropEvent>> factory) where TDropEvent : DropOperationBaseEvent
{
return HandleKnownErrors(
factory,
(errorMessage) =>
{
var dropEvent = Activator.CreateInstance<TDropEvent>();
dropEvent.Succeeded = false;
dropEvent.ErrorMessage = errorMessage;
return dropEvent;
});
}
private static async Task<TResult> HandleKnownErrors<TResult>(Func<Task<TResult>> factory, Func<string, TResult> errorValueFactory)
{
try
{
return await factory();
}
catch (VssUnauthorizedException e)
{
return errorValueFactory("[DROP AUTH ERROR] " + e.Message);
}
catch (DropServiceException e)
{
return errorValueFactory("[DROP SERVICE ERROR] " + e.Message);
}
catch (DaemonException e)
{
return errorValueFactory("[DAEMON ERROR] " + e.Message);
}
}
private static string DropItemToString(DropItem dropItem)
{
try
{
return dropItem?.ToJson().ToString();
}
#pragma warning disable ERP022 // TODO: This should really handle specific errors
catch
{
return null;
}
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
}
private static int ComputeDropItemExpiration(DropItem dropItem)
{
DateTime? expirationDate;
return dropItem.TryGetExpirationTime(out expirationDate) || expirationDate.HasValue
? (int)expirationDate.Value.Subtract(DateTime.UtcNow).TotalDays
: -1;
}
private async Task<T> SendDropEtwEvent<T>(Task<T> task) where T : DropOperationBaseEvent
{
long startTime = DateTime.UtcNow.Ticks;
T dropEvent = null;
try
{
dropEvent = await task;
return dropEvent;
}
finally
{
// if 'task' failed, create an event indicating an error
if (dropEvent == null)
{
dropEvent = Activator.CreateInstance<T>();
dropEvent.Succeeded = false;
dropEvent.ErrorMessage = "internal error";
}
// common properties: execution time, drop type, drop url
dropEvent.ElapsedTimeTicks = DateTime.UtcNow.Ticks - startTime;
dropEvent.DropType = "VsoDrop";
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
{
dropEvent.DropUrl = (await m_dropClientTask).DropUrl;
}
// send event
m_etwLogger.Log(dropEvent);
}
}
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
{
return new DropConfig(
dropName: conf.Get(DropNameOption),
serviceEndpoint: conf.Get(DropEndpoint),
maxParallelUploads: conf.Get(MaxParallelUploads),
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
verbose: conf.Get(Verbose),
enableTelemetry: conf.Get(EnableTelemetry),
enableChunkDedup: conf.Get(EnableChunkDedup),
logDir: conf.Get(LogDir));
}
private static T RegisterDropConfigOption<T>(T option) where T : Option => RegisterOption(DropConfigOptions, option);
private static Client CreateClient(string serverMoniker, IClientConfig config)
{
return serverMoniker != null
? Client.Create(serverMoniker, config)
: null;
}
private static async Task<IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
{
var files = File.GetValues(conf.Config).ToArray();
var fileIds = FileId.GetValues(conf.Config).ToArray();
var hashes = HashOptional.GetValues(conf.Config).ToArray();
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
{
return new IpcResult(
IpcResultStatus.GenericError,
I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"));
}
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
{
return new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"));
}
(Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
if (filterInitError != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, filterInitError);
}
var dropFileItemsKeyedByIsAbsent = Enumerable
.Range(0, files.Length)
.Select(i => new DropItemForBuildXLFile(
daemon.ApiClient,
chunkDedup: conf.Get(EnableChunkDedup),
filePath: files[i],
fileId: fileIds[i],
fileContentInfo: FileContentInfo.Parse(hashes[i]),
relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// If a user specified a particular file to be added to drop, this file must be a part of drop.
// The missing files will not get into the drop, so we emit an error.
if (dropFileItemsKeyedByIsAbsent[true].Any())
{
return new IpcResult(
IpcResultStatus.InvalidInput,
I($"The following files are missing, but they are a part of the drop command:{Environment.NewLine}{string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])}"));
}
(IEnumerable<DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
conf,
daemon,
directoryPaths,
directoryIds,
directoryDropPaths,
initializedFilters);
if (error != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, error);
}
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
{
return new IpcResult(
IpcResultStatus.InvalidInput,
I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"));
}
// return early if there is nothing to upload
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
{
return new IpcResult(IpcResultStatus.Success, string.Empty);
}
return await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]));
}
private static (Regex[], string error) InitializeDirectoryFilters(string[] filters)
{
try
{
var initializedFilters = filters.Select(
filter => filter == IncludeAllFilter
? null
: new Regex(filter, RegexOptions.Compiled | RegexOptions.IgnoreCase));
return (initializedFilters.ToArray(), null);
}
catch (Exception e)
{
return (null, e.ToString());
}
}
private static async Task<(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
ConfiguredCommand conf,
DropDaemon daemon,
string directoryPath,
string directoryId,
string dropPath,
Regex contentFilter)
{
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
Contract.Requires(!string.IsNullOrEmpty(directoryId));
Contract.Requires(dropPath != null);
if (daemon.ApiClient == null)
{
return (null, "ApiClient is not initialized");
}
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
if (!maybeResult.Succeeded)
{
return (null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
}
var directoryContent = maybeResult.Result;
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
if (contentFilter != null)
{
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
directoryContent = filteredContent;
}
return (directoryContent.Select(file =>
{
// we need to convert '\' into '/' because this path would be a part of a drop url
var remoteFileName = I($"{dropPath}/{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}");
return new DropItemForBuildXLFile(
daemon.ApiClient,
file.FileName,
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
conf.Get(EnableChunkDedup),
file.ContentInfo,
remoteFileName);
}).ToArray(), null);
}
private static string GetRelativePath(string root, string file)
{
var rootEndsWithSlash =
root[root.Length - 1] == System.IO.Path.DirectorySeparatorChar
|| root[root.Length - 1] == System.IO.Path.AltDirectorySeparatorChar;
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
}
private static async Task<(IEnumerable<DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
ConfiguredCommand conf,
DropDaemon daemon,
string[] directoryPaths,
string[] directoryIds,
string[] dropPaths,
Regex[] contentFilters)
{
Contract.Requires(directoryPaths != null);
Contract.Requires(directoryIds != null);
Contract.Requires(dropPaths != null);
Contract.Requires(contentFilters != null);
Contract.Requires(directoryPaths.Length == directoryIds.Length);
Contract.Requires(directoryPaths.Length == dropPaths.Length);
Contract.Requires(directoryPaths.Length == contentFilters.Length);
var createDropItemsTasks = Enumerable
.Range(0, directoryPaths.Length)
.Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();
var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);
if (createDropItemsResults.Any(r => r.error != null))
{
return (null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
}
return (createDropItemsResults.SelectMany(r => r.Item1), null);
}
private static (IEnumerable<DropItemForBuildXLFile>, string error) DedupeDropItems(IEnumerable<DropItemForBuildXLFile> dropItems)
{
var dropItemsByDropPaths = new Dictionary<string, DropItemForBuildXLFile>(StringComparer.OrdinalIgnoreCase);
foreach (var dropItem in dropItems)
{
if (dropItemsByDropPaths.TryGetValue(dropItem.RelativeDropPath, out var existingDropItem))
{
if (!string.Equals(dropItem.FullFilePath, existingDropItem.FullFilePath, StringComparison.OrdinalIgnoreCase))
{
return (
null,
I($"'{dropItem.FullFilePath}' cannot be added to drop because it has the same drop path '{dropItem.RelativeDropPath}' as '{existingDropItem.FullFilePath}'"));
}
}
else
{
dropItemsByDropPaths.Add(dropItem.RelativeDropPath, dropItem);
}
}
return (dropItemsByDropPaths.Select(kvp => kvp.Value).ToArray(), null);
}
private static async Task<IIpcResult> AddDropItemsAsync(DropDaemon daemon, IEnumerable<DropItemForBuildXLFile> dropItems)
{
(IEnumerable<DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems);
if (error != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, error);
}
var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray();
var ipcResults = await TaskUtilities.SafeWhenAll(ipcResultTasks);
return IpcResult.Merge(ipcResults);
}
}
}

11
Public/Src/Tools/DropDaemon/DropItemForBuildXLFile.cs
Просмотреть файл

@ -10,6 +10,7 @@ using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Ipc.ExternalApi;
using BuildXL.Storage;
using BuildXL.Utilities;
using Tool.ServicePipDaemon;
namespace Tool.DropDaemon
{
@ -69,29 +70,29 @@ namespace Tool.DropDaemon
/// <summary>
/// FileInfo is not already computed, sends an IPC request to BuildXL to materialize the file;
/// if the request succeeds, returns a <see cref="FileInfo"/> corresponding to that file,
/// otherwise throws a <see cref="DropDaemonException"/>.
/// otherwise throws a <see cref="DaemonException"/>.
/// </summary>
public override async Task<FileInfo> EnsureMaterialized()
{
Possible<bool> maybeResult = await m_client.MaterializeFile(m_file, FullFilePath);
if (!maybeResult.Succeeded)
{
throw new DropDaemonException(maybeResult.Failure.Describe());
throw new DaemonException(maybeResult.Failure.Describe());
}
if (!maybeResult.Result)
{
throw new DropDaemonException("File materialization failed");
throw new DaemonException("File materialization failed");
}
if (!File.Exists(FullFilePath))
{
throw new DropDaemonException("File materialization succeeded, but file is not found on disk: " + FullFilePath);
throw new DaemonException("File materialization succeeded, but file is not found on disk: " + FullFilePath);
}
if (m_symlinkTester(FullFilePath))
{
throw new DropDaemonException(MaterializationResultIsSymlinkErrorPrefix + FullFilePath);
throw new DaemonException(MaterializationResultIsSymlinkErrorPrefix + FullFilePath);
}
#if DEBUG

13
Public/Src/Tools/DropDaemon/DropItemForFile.cs
Просмотреть файл

@ -7,6 +7,7 @@ using System.Threading;
using System.Threading.Tasks;
using BuildXL.Storage;
using Microsoft.VisualStudio.Services.BlobStore.Common;
using Tool.ServicePipDaemon;
using static BuildXL.Utilities.FormattableStringEx;
namespace Tool.DropDaemon
@ -75,7 +76,7 @@ namespace Tool.DropDaemon
{
if (!File.Exists(FullFilePath))
{
throw new DropDaemonException("File not found on disk: " + FullFilePath);
throw new DaemonException("File not found on disk: " + FullFilePath);
}
return Task.FromResult(new FileInfo(FullFilePath));
@ -97,7 +98,7 @@ namespace Tool.DropDaemon
/// <summary>
/// Checks if a given <see cref="BlobIdentifier"/> matches one computed from a file on disk
/// (at <paramref name="filePath"/> location). If it doesn't, throws <see cref="DropDaemonException"/>.
/// (at <paramref name="filePath"/> location). If it doesn't, throws <see cref="DaemonException"/>.
/// </summary>
/// <remarks>
/// This method should only be called from '#if DEBUG' blocks, because it's wasteful to recompute hashes all the time.
@ -111,20 +112,20 @@ namespace Tool.DropDaemon
var calculated = (await ComputeFileDescriptorFromFileAsync(filePath, chunkDedup, Path.GetFileName(filePath), cancellationToken)).BlobIdentifier;
if (!precomputed.Equals(calculated))
{
throw new DropDaemonException(I($"[{phase}] Given blob identifier ({precomputed}) differs from computed one ({calculated}) for file '{filePath}'."));
throw new DaemonException(I($"[{phase}] Given blob identifier ({precomputed}) differs from computed one ({calculated}) for file '{filePath}'."));
}
var actualFileLength = new FileInfo(filePath).Length;
if (expectedFileLength > 0 && expectedFileLength != actualFileLength)
{
throw new DropDaemonException(I($"[{phase}] Given file length ({expectedFileLength}) differs from the file size on disk({actualFileLength}) for file '{filePath}'."));
throw new DaemonException(I($"[{phase}] Given file length ({expectedFileLength}) differs from the file size on disk({actualFileLength}) for file '{filePath}'."));
}
}
/// <summary>
/// Deserializes <see cref="BlobIdentifier"/> from a string.
///
/// Throws <see cref="DropDaemonException"/> if the string cannot be parsed.
/// Throws <see cref="DaemonException"/> if the string cannot be parsed.
/// </summary>
private static BlobIdentifier DeserializeBlobIdentifierFromHash(string serializedVsoHash)
{
@ -134,7 +135,7 @@ namespace Tool.DropDaemon
BuildXL.Cache.ContentStore.Hashing.ContentHash contentHash;
if (!BuildXL.Cache.ContentStore.Hashing.ContentHash.TryParse(serializedVsoHash, out contentHash))
{
throw new DropDaemonException("Could not parse content hash: " + serializedVsoHash);
throw new DaemonException("Could not parse content hash: " + serializedVsoHash);
}
return new BlobIdentifier(contentHash.ToHashByteArray());

20
Public/Src/Tools/DropDaemon/IDropClient.cs
Просмотреть файл

@ -10,43 +10,43 @@ using Microsoft.VisualStudio.Services.Drop.WebApi;
namespace Tool.DropDaemon
{
/// <summary>
/// Abstraction for communicating with a drop service endpoint.
/// Abstraction for communicating with a drop service endpoint.
/// </summary>
/// <remarks>
/// All the the methods/properties in this interface assume that the concrete <see cref="IDropClient"/> instance
/// has already been initialized with the necessary drop settings (<see cref="DropConfig"/>).
/// All the methods/properties in this interface assume that the concrete <see cref="IDropClient"/> instance
/// has already been initialized with the necessary drop settings (<see cref="DropConfig"/>).
/// </remarks>
public interface IDropClient : IDisposable
{
/// <summary>
/// URL at which the drop can be obtained/viewed.
/// URL at which the drop can be obtained/viewed.
/// </summary>
string DropUrl { get; }
/// <summary>
/// Task for performing 'drop create'.
/// Task for performing 'drop create'.
/// </summary>
Task<DropItem> CreateAsync();
/// <summary>
/// Task for performing 'drop addfile'.
/// Task for performing 'drop addfile'.
/// </summary>
Task<AddFileResult> AddFileAsync([NotNull]IDropItem dropItem);
/// <summary>
/// Task for performing 'drop finalize'.
/// Task for performing 'drop finalize'.
/// </summary>
Task<FinalizeResult> FinalizeAsync();
/// <summary>
/// Arbitrary statistics to report;
/// Arbitrary statistics to report;
/// </summary>
[NotNull]
IDictionary<string, long> GetStats();
}
/// <summary>
/// Result of the 'AddFile' operation (called on a single file).
/// Result of the 'AddFile' operation (called on a single file).
/// </summary>
public enum AddFileResult
{
@ -58,7 +58,7 @@ namespace Tool.DropDaemon
}
/// <summary>
/// Placeholder for future result that might be returned by <see cref="IDropClient.FinalizeAsync"/>.
/// Placeholder for future result that might be returned by <see cref="IDropClient.FinalizeAsync"/>.
/// </summary>
public sealed class FinalizeResult
{ }

18
Public/Src/Tools/DropDaemon/IDropItem.cs
Просмотреть файл

@ -11,42 +11,42 @@ using Pure = System.Diagnostics.Contracts.PureAttribute;
namespace Tool.DropDaemon
{
/// <summary>
/// Item to be added to drop.
/// Item to be added to drop.
/// </summary>
public interface IDropItem
{
/// <summary>
/// Full path of the file on disk to be added to drop. The file need not be
/// physically present on disk before <see cref="EnsureMaterialized"/> is called.
/// Full path of the file on disk to be added to drop. The file need not be
/// physically present on disk before <see cref="EnsureMaterialized"/> is called.
/// </summary>
[NotNull]
[Pure]
string FullFilePath { get; }
/// <summary>
/// Relative path under which to associate the file with a drop.
/// Relative path under which to associate the file with a drop.
/// </summary>
[NotNull]
[Pure]
string RelativeDropPath { get; }
/// <summary>
/// (Optional) Pre-computed blob identifier.
/// (Optional) Pre-computed blob identifier.
/// </summary>
[CanBeNull]
[Pure]
BlobIdentifier BlobIdentifier { get; }
/// <summary>
/// (Optional) Pre-computed file length.
/// (Optional) Pre-computed file length.
/// </summary>
[Pure]
long FileLength { get; }
/// <summary>
/// Full information about the file which is to be added to a drop. After the completion
/// of the returned task, the file must exist on disk. The returned file info must also
/// match the full file path returned by the <see cref="FullFilePath"/> property.
/// Full information about the file which is to be added to a drop. After the completion
/// of the returned task, the file must exist on disk. The returned file info must also
/// match the full file path returned by the <see cref="FullFilePath"/> property.
/// </summary>
[NotNull]
Task<FileInfo> EnsureMaterialized();

908
Public/Src/Tools/DropDaemon/Program.cs
Просмотреть файл

@ -2,653 +2,21 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Ipc;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.ExternalApi;
using BuildXL.Ipc.Interfaces;
using BuildXL.Scheduler;
using BuildXL.Storage;
using BuildXL.Utilities;
using BuildXL.Utilities.CLI;
using BuildXL.Utilities.Tasks;
using Newtonsoft.Json.Linq;
using static Tool.DropDaemon.Statics;
using Tool.ServicePipDaemon;
using static Tool.ServicePipDaemon.Statics;
namespace Tool.DropDaemon
{
/// <summary>
/// DropDaemon entry point.
/// DropDaemon entry point.
/// </summary>
public static class Program
{
internal static IIpcProvider IpcProvider = IpcFactory.GetProvider();
internal static readonly List<Option> DaemonConfigOptions = new List<Option>();
internal static readonly List<Option> DropConfigOptions = new List<Option>();
private const int ServicePointParallelismForDrop = 200;
private static readonly int s_minIoThreadsForDrop = Environment.ProcessorCount * 10;
private static readonly int s_minWorkerThreadsForDrop = Environment.ProcessorCount * 10;
private const string IncludeAllFilter = ".*";
// ==============================================================================
// Daemon config
// ==============================================================================
internal static readonly StrOption Moniker = RegisterDaemonConfigOption(new StrOption("moniker")
{
ShortName = "m",
HelpText = "Moniker to identify client/server communication",
});
internal static readonly IntOption MaxConcurrentClients = RegisterDaemonConfigOption(new IntOption("maxConcurrentClients")
{
HelpText = "OBSOLETE due to the hardcoded config. (Maximum number of clients to serve concurrently)",
DefaultValue = DaemonConfig.DefaultMaxConcurrentClients,
});
internal static readonly IntOption MaxConnectRetries = RegisterDaemonConfigOption(new IntOption("maxConnectRetries")
{
HelpText = "Maximum number of retries to establish a connection with a running daemon",
DefaultValue = DaemonConfig.DefaultMaxConnectRetries,
});
internal static readonly IntOption ConnectRetryDelayMillis = RegisterDaemonConfigOption(new IntOption("connectRetryDelayMillis")
{
HelpText = "Delay between consecutive retries to establish a connection with a running daemon",
DefaultValue = (int)DaemonConfig.DefaultConnectRetryDelay.TotalMilliseconds,
});
internal static readonly BoolOption ShellExecute = RegisterDaemonConfigOption(new BoolOption("shellExecute")
{
HelpText = "Use shell execute to start the daemon process (a shell window will be created and displayed)",
DefaultValue = false,
});
internal static readonly BoolOption StopOnFirstFailure = RegisterDaemonConfigOption(new BoolOption("stopOnFirstFailure")
{
HelpText = "Daemon process should terminate after first failed operation (e.g., 'drop create' fails because the drop already exists).",
DefaultValue = DaemonConfig.DefaultStopOnFirstFailure,
});
internal static readonly BoolOption EnableCloudBuildIntegration = RegisterDaemonConfigOption(new BoolOption("enableCloudBuildIntegration")
{
ShortName = "ecb",
HelpText = "Enable logging ETW events for CloudBuild to pick up",
DefaultValue = DaemonConfig.DefaultEnableCloudBuildIntegration,
});
internal static readonly BoolOption Verbose = RegisterDaemonConfigOption(new BoolOption("verbose")
{
ShortName = "v",
HelpText = "Verbose logging",
IsRequired = false,
DefaultValue = DropConfig.DefaultVerbose,
});
internal static readonly StrOption DropServiceConfigFile = RegisterDaemonConfigOption(new StrOption("dropServiceConfigFile")
{
ShortName = "c",
HelpText = "Drop service configuration file",
DefaultValue = null,
Expander = (fileName) =>
{
var json = System.IO.File.ReadAllText(fileName);
var jObject = JObject.Parse(json);
return jObject.Properties().Select(prop => new ParsedOption(PrefixKind.Long, prop.Name, prop.Value.ToString()));
},
});
internal static readonly StrOption LogDir = RegisterDaemonConfigOption(new StrOption("logDir")
{
ShortName = "log",
HelpText = "Log directory",
IsRequired = false
});
// ==============================================================================
// Drop config
// ==============================================================================
internal static readonly StrOption DropName = RegisterDropConfigOption(new StrOption("name")
{
ShortName = "n",
HelpText = "Drop name",
IsRequired = true,
});
internal static readonly UriOption DropEndpoint = RegisterDropConfigOption(new UriOption("service")
{
ShortName = "s",
HelpText = "Drop endpoint URI",
IsRequired = true,
});
internal static readonly IntOption BatchSize = RegisterDropConfigOption(new IntOption("batchSize")
{
ShortName = "bs",
HelpText = "OBSOLETE due to the hardcoded config. (Size of batches in which to send 'associate' requests)",
IsRequired = false,
DefaultValue = DropConfig.DefaultBatchSizeForAssociate,
});
internal static readonly IntOption MaxParallelUploads = RegisterDropConfigOption(new IntOption("maxParallelUploads")
{
ShortName = "mpu",
HelpText = "Maximum number of uploads to issue to drop service in parallel",
IsRequired = false,
DefaultValue = DropConfig.DefaultMaxParallelUploads,
});
internal static readonly IntOption NagleTimeMillis = RegisterDropConfigOption(new IntOption("nagleTimeMillis")
{
ShortName = "nt",
HelpText = "OBSOLETE due to the hardcoded config. (Maximum time in milliseconds to wait before triggering a batch 'associate' request)",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultNagleTimeForAssociate.TotalMilliseconds,
});
internal static readonly IntOption RetentionDays = RegisterDropConfigOption(new IntOption("retentionDays")
{
ShortName = "rt",
HelpText = "Drop retention time in days",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultRetention.TotalDays,
});
internal static readonly IntOption HttpSendTimeoutMillis = RegisterDropConfigOption(new IntOption("httpSendTimeoutMillis")
{
HelpText = "Timeout for http requests",
IsRequired = false,
DefaultValue = (int)DropConfig.DefaultHttpSendTimeout.TotalMilliseconds,
});
internal static readonly BoolOption EnableTelemetry = RegisterDropConfigOption(new BoolOption("enableTelemetry")
{
ShortName = "t",
HelpText = "Verbose logging",
IsRequired = false,
DefaultValue = DropConfig.DefaultEnableTelemetry,
});
internal static readonly BoolOption EnableChunkDedup = RegisterDropConfigOption(new BoolOption("enableChunkDedup")
{
ShortName = "cd",
HelpText = "Chunk level dedup",
IsRequired = false,
DefaultValue = DropConfig.DefaultEnableChunkDedup,
});
// ==============================================================================
// 'addfile' and 'addartifacts' parameters
// ==============================================================================
internal static readonly StrOption RelativeDropPath = new StrOption("dropPath")
{
ShortName = "d",
HelpText = "Relative drop path",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption File = new StrOption("file")
{
ShortName = "f",
HelpText = "File path",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption HashOptional = new StrOption("hash")
{
ShortName = "h",
HelpText = "VSO file hash",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption FileId = new StrOption("fileId")
{
ShortName = "fid",
HelpText = "BuildXL file identifier",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption IpcServerMonikerRequired = new StrOption("ipcServerMoniker")
{
ShortName = "dm",
HelpText = "IPC moniker identifying a running BuildXL IPC server",
IsRequired = true,
};
internal static readonly StrOption HelpNoNameOption = new StrOption(string.Empty)
{
HelpText = "Command name",
};
internal static readonly StrOption Directory = new StrOption("directory")
{
ShortName = "dir",
HelpText = "Directory path",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption DirectoryId = new StrOption("directoryId")
{
ShortName = "dirid",
HelpText = "BuildXL directory identifier",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption RelativeDirectoryDropPath = new StrOption("directoryDropPath")
{
ShortName = "dird",
HelpText = "Relative drop path for directory",
IsRequired = false,
IsMultiValue = true,
};
internal static readonly StrOption DirectoryContentFilter = new StrOption("directoryFilter")
{
ShortName = "dcfilter",
HelpText = "Directory content filter (only files that match the filter will be added to drop).",
DefaultValue = null,
IsRequired = false,
IsMultiValue = false,
};
// ==============================================================================
// Commands
// ==============================================================================
internal static readonly Dictionary<string, Command> Commands = new Dictionary<string, Command>();
/// <remarks>
/// The <see cref="DaemonConfigOptions"/> options are added to every command.
/// A non-mandatory string option "name" is added as well, which drop operation
/// commands may want to use to explicitly specify the target drop name.
/// </remarks>
private static Command RegisterCommand(
string name,
IEnumerable<Option> options = null,
ServerAction serverAction = null,
ClientAction clientAction = null,
string description = null,
bool needsIpcClient = true,
bool addDaemonConfigOptions = true)
{
var opts = (options ?? new Option[0]).ToList();
if (addDaemonConfigOptions)
{
opts.AddRange(DaemonConfigOptions);
}
if (!opts.Exists(opt => opt.LongName == "name"))
{
opts.Add(new Option(longName: DropName.LongName)
{
ShortName = DropName.ShortName,
});
}
var cmd = new Command(name, opts, serverAction, clientAction, description, needsIpcClient);
Commands[cmd.Name] = cmd;
return cmd;
}
internal static readonly Command HelpCmd = RegisterCommand(
name: "help",
description: "Prints a help message (usage).",
options: new[] { HelpNoNameOption },
needsIpcClient: false,
clientAction: (conf, rpc) =>
{
string cmdName = conf.Get(HelpNoNameOption);
bool cmdNotSpecified = string.IsNullOrWhiteSpace(cmdName);
if (cmdNotSpecified)
{
Console.WriteLine(Usage());
return 0;
}
Command requestedHelpForCommand;
var requestedCommandFound = Commands.TryGetValue(cmdName, out requestedHelpForCommand);
if (requestedCommandFound)
{
Console.WriteLine(requestedHelpForCommand.Usage(conf.Config.Parser));
return 0;
}
else
{
Console.WriteLine(Usage());
return 1;
}
});
internal static readonly Command StartNoDropCmd = RegisterCommand(
name: "start-nodrop",
description: @"Starts a server process without a backing VSO drop client (useful for testing/pinging the daemon).",
needsIpcClient: false,
clientAction: (conf, _) =>
{
var dropConfig = new DropConfig(string.Empty, new Uri("file://xyz"));
var daemonConfig = CreateDaemonConfig(conf);
var vsoClientTask = TaskSourceSlim.Create<IDropClient>();
vsoClientTask.SetException(new NotSupportedException());
using (var daemon = new Daemon(conf.Config.Parser, daemonConfig, dropConfig, vsoClientTask.Task))
{
daemon.Start();
daemon.Completion.GetAwaiter().GetResult();
return 0;
}
});
internal static readonly StrOption IpcServerMonikerOptional = new StrOption(
longName: IpcServerMonikerRequired.LongName)
{
ShortName = IpcServerMonikerRequired.ShortName,
HelpText = IpcServerMonikerRequired.HelpText,
IsRequired = false,
};
internal static readonly Command StartCmd = RegisterCommand(
name: "start",
description: "Starts the server process.",
options: DropConfigOptions.Union(new[] { IpcServerMonikerOptional }),
needsIpcClient: false,
clientAction: (conf, _) =>
{
SetupThreadPoolAndServicePoint();
var dropConfig = CreateDropConfig(conf);
var daemonConf = CreateDaemonConfig(conf);
if (daemonConf.MaxConcurrentClients <= 1)
{
conf.Logger.Error($"Must specify at least 2 '{nameof(DaemonConfig.MaxConcurrentClients)}' when running DropDaemon to avoid deadlock when stopping this daemon from a different client");
return -1;
}
using (var client = CreateClient(conf.Get(IpcServerMonikerOptional), daemonConf))
using (var daemon = new Daemon(
parser: conf.Config.Parser,
daemonConfig: daemonConf,
dropConfig: dropConfig,
dropClientTask: null,
client: client))
{
daemon.Start();
daemon.Completion.GetAwaiter().GetResult();
return 0;
}
});
private static void SetupThreadPoolAndServicePoint()
{
int workerThreads, ioThreads;
ThreadPool.GetMinThreads(out workerThreads, out ioThreads);
workerThreads = Math.Max(workerThreads, s_minWorkerThreadsForDrop);
ioThreads = Math.Max(ioThreads, s_minIoThreadsForDrop);
ThreadPool.SetMinThreads(workerThreads, ioThreads);
ServicePointManager.DefaultConnectionLimit = Math.Max(ServicePointParallelismForDrop, ServicePointManager.DefaultConnectionLimit);
}
internal static readonly Command StartDaemonCmd = RegisterCommand(
name: "start-daemon",
description: "Starts the server process in background (as daemon).",
options: DropConfigOptions,
needsIpcClient: false,
clientAction: (conf, _) =>
{
using (var daemon = new Process())
{
bool shellExecute = conf.Get(ShellExecute);
daemon.StartInfo.FileName = AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetEntryAssembly());
daemon.StartInfo.Arguments = "start " + conf.Config.Render();
daemon.StartInfo.LoadUserProfile = false;
daemon.StartInfo.UseShellExecute = shellExecute;
daemon.StartInfo.CreateNoWindow = !shellExecute;
daemon.Start();
}
return 0;
});
internal static readonly Command StopDaemonCmd = RegisterCommand(
name: "stop",
description: "[RPC] Stops the daemon process running on specified port; fails if no such daemon is running.",
clientAction: AsyncRPCSend,
serverAction: (conf, daemon) =>
{
conf.Logger.Info("[STOP] requested");
daemon.RequestStop();
return Task.FromResult(IpcResult.Success());
});
internal static readonly Command CrashDaemonCmd = RegisterCommand(
name: "crash",
description: "[RPC] Stops the server process by crashing it.",
clientAction: AsyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[CRASH] requested");
Environment.Exit(-1);
return Task.FromResult(IpcResult.Success());
});
internal static readonly Command PingDaemonCmd = RegisterCommand(
name: "ping",
description: "[RPC] Pings the daemon process.",
clientAction: SyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[PING] received");
return Task.FromResult(IpcResult.Success("Alive!"));
});
internal static readonly Command CreateDropCmd = RegisterCommand(
name: "create",
description: "[RPC] Invokes the 'create' operation.",
options: DropConfigOptions,
clientAction: SyncRPCSend,
serverAction: async (conf, daemon) =>
{
daemon.Logger.Info("[CREATE]: Started at " + daemon.DropConfig.Service + "/" + daemon.DropName);
IIpcResult result = await daemon.CreateAsync();
daemon.Logger.Info("[CREATE]: " + result);
return result;
});
internal static readonly Command FinalizeDropCmd = RegisterCommand(
name: "finalize",
description: "[RPC] Invokes the 'finalize' operation.",
clientAction: SyncRPCSend,
serverAction: async (conf, daemon) =>
{
daemon.Logger.Info("[FINALIZE] Started at" + daemon.DropConfig.Service + "/" + daemon.DropName);
IIpcResult result = await daemon.FinalizeAsync();
daemon.Logger.Info("[FINALIZE] " + result);
return result;
});
internal static readonly Command FinalizeDropAndStopDaemonCmd = RegisterCommand(
name: "finalize-and-stop",
description: "[RPC] Invokes the 'finalize' operation; then stops the daemon.",
clientAction: SyncRPCSend,
serverAction: Command.Compose(FinalizeDropCmd.ServerAction, StopDaemonCmd.ServerAction));
internal static readonly Command AddFileToDropCmd = RegisterCommand(
name: "addfile",
description: "[RPC] invokes the 'addfile' operation.",
options: new Option[] { File, RelativeDropPath, HashOptional },
clientAction: SyncRPCSend,
serverAction: async (conf, daemon) =>
{
daemon.Logger.Verbose("[ADDFILE] Started");
string filePath = conf.Get(File);
string hashValue = conf.Get(HashOptional);
var contentInfo = string.IsNullOrEmpty(hashValue) ? null : (FileContentInfo?)FileContentInfo.Parse(hashValue);
var dropItem = new DropItemForFile(filePath, conf.Get(RelativeDropPath), contentInfo);
IIpcResult result = System.IO.File.Exists(filePath)
? await daemon.AddFileAsync(dropItem)
: new IpcResult(IpcResultStatus.ExecutionError, "file '" + filePath + "' does not exist");
daemon.Logger.Verbose("[ADDFILE] " + result);
return result;
});
internal static readonly Command AddArtifactsToDropCmd = RegisterCommand(
name: "addartifacts",
description: "[RPC] invokes the 'addartifacts' operation.",
options: new Option[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath, Directory, DirectoryId, RelativeDirectoryDropPath, DirectoryContentFilter },
clientAction: SyncRPCSend,
serverAction: async (conf, daemon) =>
{
daemon.Logger.Verbose("[ADDARTIFACTS] Started");
var result = await AddArtifactsToDropInternalAsync(conf, daemon);
daemon.Logger.Verbose("[ADDARTIFACTS] " + result);
return result;
});
private static async Task<IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, Daemon daemon)
{
var files = File.GetValues(conf.Config).ToArray();
var fileIds = FileId.GetValues(conf.Config).ToArray();
var hashes = HashOptional.GetValues(conf.Config).ToArray();
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
{
return new IpcResult(
IpcResultStatus.GenericError,
Inv(
"File counts don't match: #files = {0}, #fileIds = {1}, #hashes = {2}, #dropPaths = {3}",
files.Length, fileIds.Length, hashes.Length, dropPaths.Length));
}
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
{
return new IpcResult(
IpcResultStatus.GenericError,
Inv(
"Directory counts don't match: #directories = {0}, #directoryIds = {1}, #dropPaths = {2}, #directoryFilters = {3}",
directoryPaths.Length, directoryIds.Length, directoryDropPaths.Length, directoryFilters.Length));
}
(Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
if (filterInitError != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, filterInitError);
}
var dropFileItemsKeyedByIsAbsent = Enumerable
.Range(0, files.Length)
.Select(i => new DropItemForBuildXLFile(
daemon.ApiClient,
chunkDedup: conf.Get(EnableChunkDedup),
filePath: files[i],
fileId: fileIds[i],
fileContentInfo: FileContentInfo.Parse(hashes[i]),
relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// If a user specified a particular file to be added to drop, this file must be a part of drop.
// The missing files will not get into the drop, so we emit an error.
if (dropFileItemsKeyedByIsAbsent[true].Any())
{
return new IpcResult(
IpcResultStatus.InvalidInput,
Inv("The following files are missing, but they are a part of the drop command:{0}{1}",
Environment.NewLine,
string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])));
}
(IEnumerable<DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
conf,
daemon,
directoryPaths,
directoryIds,
directoryDropPaths,
initializedFilters);
if (error != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, error);
}
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
{
return new IpcResult(
IpcResultStatus.InvalidInput,
Inv("Uploading missing source file(s) is not supported:{0}{1}",
Environment.NewLine,
string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))));
}
// return early if there is nothing to upload
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
{
return new IpcResult(IpcResultStatus.Success, string.Empty);
}
return await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]));
}
private static (Regex[], string error) InitializeDirectoryFilters(string[] filters)
{
try
{
var initializedFilters = filters.Select(
filter => filter == IncludeAllFilter
? null
: new Regex(filter, RegexOptions.Compiled | RegexOptions.IgnoreCase));
return (initializedFilters.ToArray(), null);
}
catch (Exception e)
{
return (null, e.ToString());
}
}
internal static readonly Command TestReadFile = RegisterCommand(
name: "test-readfile",
description: "[RPC] Sends a request to the daemon to read a file.",
options: new Option[] { File },
clientAction: SyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[READFILE] received");
var result = IpcResult.Success(System.IO.File.ReadAllText(conf.Get(File)));
daemon.Logger.Info("[READFILE] succeeded");
return Task.FromResult(result);
});
private static Client CreateClient(string serverMoniker, IClientConfig config)
{
return serverMoniker != null
? Client.Create(serverMoniker, config)
: null;
}
/// <nodoc/>
[SuppressMessage("Microsoft.Naming", "CA2204:Spelling of DropD")]
public static int Main(string[] args)
@ -658,17 +26,19 @@ namespace Tool.DropDaemon
try
{
Console.WriteLine("DropDaemon started at " + DateTime.UtcNow);
Console.WriteLine(Daemon.DropDLogPrefix + "Command line arguments: ");
Console.WriteLine(string.Join(Environment.NewLine + Daemon.DropDLogPrefix, args));
Console.WriteLine(nameof(DropDaemon) + " started at " + DateTime.UtcNow);
Console.WriteLine(DropDaemon.DropDLogPrefix + "Command line arguments: ");
Console.WriteLine(string.Join(Environment.NewLine + DropDaemon.DropDLogPrefix, args));
Console.WriteLine();
ConfiguredCommand conf = ParseArgs(args, new UnixParser());
if (conf.Command.NeedsIpcClient)
DropDaemon.EnsureCommandsInitialized();
var confCommand = ServicePipDaemon.ServicePipDaemon.ParseArgs(args, new UnixParser());
if (confCommand.Command.NeedsIpcClient)
{
using (var rpc = CreateClient(conf))
using (var rpc = CreateClient(confCommand))
{
var result = conf.Command.ClientAction(conf, rpc);
var result = confCommand.Command.ClientAction(confCommand, rpc);
rpc.RequestStop();
rpc.Completion.GetAwaiter().GetResult();
return result;
@ -676,7 +46,7 @@ namespace Tool.DropDaemon
}
else
{
return conf.Command.ClientAction(conf, null);
return confCommand.Command.ClientAction(confCommand, null);
}
}
catch (ArgumentException e)
@ -686,258 +56,10 @@ namespace Tool.DropDaemon
}
}
internal static ConfiguredCommand ParseArgs(string allArgs, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
{
return ParseArgs(parser.SplitArgs(allArgs), parser, logger, ignoreInvalidOptions);
}
internal static ConfiguredCommand ParseArgs(string[] args, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
{
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
if (args.Length == 0)
{
throw new ArgumentException(Inv("Command is required. {0}", usageMessage.Value));
}
var argsQueue = new Queue<string>(args);
string cmdName = argsQueue.Dequeue();
if (!Commands.TryGetValue(cmdName, out Command cmd))
{
throw new ArgumentException(Inv("No command '{0}' is found. {1}", cmdName, usageMessage.Value));
}
var sw = Stopwatch.StartNew();
Config conf = Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
var parseTime = sw.Elapsed;
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), Daemon.DropDLogPrefix);
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
return new ConfiguredCommand(cmd, conf, logger);
}
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
{
return new DropConfig(
dropName: conf.Get(DropName),
serviceEndpoint: conf.Get(DropEndpoint),
maxParallelUploads: conf.Get(MaxParallelUploads),
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
verbose: conf.Get(Verbose),
enableTelemetry: conf.Get(EnableTelemetry),
enableChunkDedup: conf.Get(EnableChunkDedup),
logDir: conf.Get(LogDir));
}
internal static DaemonConfig CreateDaemonConfig(ConfiguredCommand conf)
{
return new DaemonConfig(
logger: conf.Logger,
moniker: conf.Get(Moniker),
maxConnectRetries: conf.Get(MaxConnectRetries),
connectRetryDelay: TimeSpan.FromMilliseconds(conf.Get(ConnectRetryDelayMillis)),
stopOnFirstFailure: conf.Get(StopOnFirstFailure),
enableCloudBuildIntegration: conf.Get(EnableCloudBuildIntegration));
}
internal static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = CreateDaemonConfig(conf);
return IpcProvider.GetClient(daemonConfig.Moniker, daemonConfig);
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
return IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig);
}
private static async Task<(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
ConfiguredCommand conf,
Daemon daemon,
string directoryPath,
string directoryId,
string dropPath,
Regex contentFilter)
{
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
Contract.Requires(!string.IsNullOrEmpty(directoryId));
Contract.Requires(dropPath != null);
if (daemon.ApiClient == null)
{
return (null, "ApiClient is not initialized");
}
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
if (!maybeResult.Succeeded)
{
return (null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
}
var directoryContent = maybeResult.Result;
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
if (contentFilter != null)
{
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
directoryContent = filteredContent;
}
return (directoryContent.Select(file =>
{
var remoteFileName = Inv(
"{0}/{1}",
dropPath,
// we need to convert '\' into '/' because this path would be a part of a drop url
GetRelativePath(directoryPath, file.FileName).Replace('\\', '/'));
return new DropItemForBuildXLFile(
daemon.ApiClient,
file.FileName,
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
conf.Get(EnableChunkDedup),
file.ContentInfo,
remoteFileName);
}).ToArray(), null);
}
private static async Task<(IEnumerable<DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
ConfiguredCommand conf,
Daemon daemon,
string[] directoryPaths,
string[] directoryIds,
string[] dropPaths,
Regex[] contentFilters)
{
Contract.Requires(directoryPaths != null);
Contract.Requires(directoryIds != null);
Contract.Requires(dropPaths != null);
Contract.Requires(contentFilters != null);
Contract.Requires(directoryPaths.Length == directoryIds.Length);
Contract.Requires(directoryPaths.Length == dropPaths.Length);
Contract.Requires(directoryPaths.Length == contentFilters.Length);
var createDropItemsTasks = Enumerable
.Range(0, directoryPaths.Length)
.Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();
var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);
if (createDropItemsResults.Any(r => r.error != null))
{
return (null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
}
return (createDropItemsResults.SelectMany(r => r.Item1), null);
}
private static string GetRelativePath(string root, string file)
{
var rootEndsWithSlash =
root[root.Length - 1] == System.IO.Path.DirectorySeparatorChar
|| root[root.Length - 1] == System.IO.Path.AltDirectorySeparatorChar;
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
}
private static (IEnumerable<DropItemForBuildXLFile>, string error) DedupeDropItems(IEnumerable<DropItemForBuildXLFile> dropItems)
{
var dropItemsByDropPaths = new Dictionary<string, DropItemForBuildXLFile>(StringComparer.OrdinalIgnoreCase);
foreach (var dropItem in dropItems)
{
if (dropItemsByDropPaths.TryGetValue(dropItem.RelativeDropPath, out var existingDropItem))
{
if (!string.Equals(dropItem.FullFilePath, existingDropItem.FullFilePath, StringComparison.OrdinalIgnoreCase))
{
return (
null,
Inv(
"'{0}' cannot be added to drop because it has the same drop path '{1}' as '{2}'",
dropItem.FullFilePath,
dropItem.RelativeDropPath,
existingDropItem.FullFilePath));
}
}
else
{
dropItemsByDropPaths.Add(dropItem.RelativeDropPath, dropItem);
}
}
return (dropItemsByDropPaths.Select(kvp => kvp.Value).ToArray(), null);
}
private static async Task<IIpcResult> AddDropItemsAsync(Daemon daemon, IEnumerable<DropItemForBuildXLFile> dropItems)
{
(IEnumerable<DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems);
if (error != null)
{
return new IpcResult(IpcResultStatus.ExecutionError, error);
}
var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray();
var ipcResults = await TaskUtilities.SafeWhenAll(ipcResultTasks);
return IpcResult.Merge(ipcResults);
}
private static string Usage()
{
var builder = new StringBuilder();
var len = Commands.Keys.Max(cmdName => cmdName.Length);
foreach (var cmd in Commands.Values)
{
builder.AppendLine(Inv(" {0,-" + len + "} : {1}", cmd.Name, cmd.Description));
}
return builder.ToString();
}
private static int SyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, true);
private static int AsyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, false);
private static int RPCSend(ConfiguredCommand conf, IClient rpc, bool isSync)
{
var rpcResult = RPCSendCore(conf, rpc, isSync);
conf.Logger.Info(
"Command '{0}' {1} (exit code: {2}). {3}",
conf.Command.Name,
rpcResult.Succeeded ? "succeeded" : "failed",
(int)rpcResult.ExitCode,
rpcResult.Payload);
return (int)rpcResult.ExitCode;
}
private static IIpcResult RPCSendCore(ConfiguredCommand conf, IClient rpc, bool isSync)
{
string operationPayload = ToPayload(conf);
var operation = new IpcOperation(operationPayload, waitForServerAck: isSync);
return rpc.Send(operation).GetAwaiter().GetResult();
}
/// <summary>
/// Reconstructs a full command line from a command name (<paramref name="commandName"/>)
/// and a configuration (<paramref name="config"/>).
/// </summary>
internal static string ToPayload(string commandName, Config config)
{
return commandName + " " + config.Render();
}
/// <summary>
/// Reconstructs a full command line corresponding to a <see cref="ConfiguredCommand"/>.
/// </summary>
private static string ToPayload(ConfiguredCommand cmd) => ToPayload(cmd.Command.Name, cmd.Config);
private static T RegisterOption<T>(List<Option> options, T option) where T : Option
{
options.Add(option);
return option;
}
private static T RegisterDaemonConfigOption<T>(T option) where T : Option => RegisterOption(DaemonConfigOptions, option);
private static T RegisterDropConfigOption<T>(T option) where T : Option => RegisterOption(DropConfigOptions, option);
}
}

1
Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc
Просмотреть файл

@ -29,6 +29,7 @@ export namespace DropDaemon {
importFrom("BuildXL.Utilities").Ipc.dll,
importFrom("BuildXL.Utilities").Native.dll,
importFrom("BuildXL.Utilities").Storage.dll,
importFrom("BuildXL.Tools").ServicePipDaemon.dll,
importFrom("ArtifactServices.App.Shared").pkg,
importFrom("ArtifactServices.App.Shared.Cache").pkg,

5
Public/Src/Tools/DropDaemon/VsoClient.cs
Просмотреть файл

@ -26,6 +26,7 @@ using Microsoft.VisualStudio.Services.Drop.WebApi;
using Microsoft.VisualStudio.Services.ItemStore.Common;
using Newtonsoft.Json;
using static BuildXL.Utilities.FormattableStringEx;
using Tool.ServicePipDaemon;
namespace Tool.DropDaemon
{
@ -362,7 +363,7 @@ namespace Tool.DropDaemon
var notFoundMissingHashes = associateStatus.Missing.Where(b => !providedHashes.Contains(b)).ToList();
if (notFoundMissingHashes.Any())
{
throw new DropDaemonException("This many hashes not found in blobs to upload: " + notFoundMissingHashes.Count());
throw new DaemonException("This many hashes not found in blobs to upload: " + notFoundMissingHashes.Count());
}
#endif
@ -482,7 +483,7 @@ namespace Tool.DropDaemon
if (m_fileBlobDescriptorForAssociate != null &&
!m_fileBlobDescriptorForAssociate.BlobIdentifier.Equals(m_fileBlobDescriptorForUpload.BlobIdentifier))
{
throw new DropDaemonException(I($"Blob identifier for file '{m_fileBlobDescriptorForUpload.AbsolutePath}' returned for 'UploadAndAssociate' ({m_fileBlobDescriptorForUpload.BlobIdentifier}) is different from the one returned for 'Associate' ({m_fileBlobDescriptorForAssociate.BlobIdentifier})."));
throw new DaemonException(I($"Blob identifier for file '{m_fileBlobDescriptorForUpload.AbsolutePath}' returned for 'UploadAndAssociate' ({m_fileBlobDescriptorForUpload.BlobIdentifier}) is different from the one returned for 'Associate' ({m_fileBlobDescriptorForAssociate.BlobIdentifier})."));
}
}

2
Public/Src/Tools/DropDaemon/BuildXLBasedCloudBuildLogger.cs → Public/Src/Tools/ServicePipDaemon/BuildXLBasedCloudBuildLogger.cs
Просмотреть файл

@ -8,7 +8,7 @@ using BuildXL.Tracing.CloudBuild;
using BuildXL.Utilities.Instrumentation.Common;
using System.Diagnostics.Tracing;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Implementation of <see cref="ICloudBuildLogger"/> that uses BuildXL's LoggingContext/>.

149
Public/Src/Tools/ServicePipDaemon/Command.cs Normal file
Просмотреть файл

@ -0,0 +1,149 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.Interfaces;
using BuildXL.Utilities.CLI;
namespace Tool.ServicePipDaemon
{
/// <nodoc/>
public delegate int ClientAction(ConfiguredCommand conf, IClient rpc);
/// <nodoc/>
public delegate Task<IIpcResult> ServerAction(ConfiguredCommand conf, ServicePipDaemon daemon);
/// <summary>
/// A command has a name, description, a list of options it supports, and two actions:
/// one for executing this command on the client, and one for executing it on the server.
///
/// When an instance of a daemon (e.g., DropDaemon) is invoked, command line arguments
/// are parsed to determine the command specified by the user. That command is then
/// interpreted by executing its <see cref="ClientAction"/>.
///
/// Most of the daemon commands will be RPC calls, i.e., when a command is received via
/// the command line, it is to be marshaled and sent over to a running daemon server via an RPC.
/// In such a case, the client action simply invokes <see cref="IClient.Send"/>.
///
/// When an RPC is received by a daemon server (<see cref="ServicePipDaemon.ParseAndExecuteCommand"/>),
/// a <see cref="Command"/> is unmarshaled from the payload of the RPC operation and
/// is interpreted on the server by executing its <see cref="ServerAction"/>.
/// </summary>
/// <remarks>
/// Immutable.
/// </remarks>
public sealed class Command
{
/// <summary>A unique command name.</summary>
public string Name { get; }
/// <summary>Arbitrary description.</summary>
public string Description { get; }
/// <summary>Options that may/must be passed to this command.</summary>
public IReadOnlyCollection<Option> Options { get; }
/// <summary>Action to be executed when this command is received via the command line.</summary>
public ClientAction ClientAction { get; }
/// <summary>Action to be executed when this command is received via an RPC call.</summary>
public ServerAction ServerAction { get; }
/// <summary>Whether this command requires an IpcClient; defaults to true.</summary>
public bool NeedsIpcClient { get; }
/// <nodoc />
public Command(
string name,
IEnumerable<Option> options = null,
ServerAction serverAction = null,
ClientAction clientAction = null,
string description = null,
bool needsIpcClient = true)
{
Contract.Requires(name != null);
Name = name;
ServerAction = serverAction;
ClientAction = clientAction;
Description = description;
Options = options.ToList();
NeedsIpcClient = needsIpcClient;
}
/// <summary>
/// Performs a functional composition of a number of <see cref="ServerAction"/> functions,
/// where the results are merged by calling <see cref="IpcResult.Merge(IIpcResult, IIpcResult)"/>.
/// </summary>
public static ServerAction Compose(params ServerAction[] actions)
{
Contract.Requires(actions != null);
Contract.Requires(actions.Length > 0);
var first = actions.First();
return actions.Skip(1).Aggregate(first, (accumulator, currentAction) => new ServerAction(async (conf, daemon) =>
{
var lhsResult = await accumulator(conf, daemon);
var rhsResult = await currentAction(conf, daemon);
return IpcResult.Merge(lhsResult, rhsResult);
}));
}
/// <summary>
/// Returns the usage information for this command.
/// </summary>
public string Usage(IParser parser)
{
var result = new StringBuilder();
var tab = " ";
result.AppendLine("NAME");
result.Append(tab).Append(Name).Append(" - ").AppendLine(Description);
result.AppendLine();
result.AppendLine("SWITCHES");
var optsSorted = Options
.OrderBy(o => o.IsRequired ? 0 : 1)
.ThenBy(o => o.LongName)
.ToArray();
result.AppendLine(parser.Usage(optsSorted, tab, tab));
return result.ToString();
}
}
/// <summary>
/// Simple wrapper class that holds a <see cref="Command"/> and a <see cref="Config"/>
/// containing actual values for the command's <see cref="Command.Options"/>.
/// </summary>
public sealed class ConfiguredCommand
{
/// <summary>
/// The command.
/// </summary>
public Command Command { get; }
/// <summary>
/// Configured values for command's options
/// </summary>
public Config Config { get; }
/// <nodoc/>
public ILogger Logger { get; }
/// <nodoc/>
public ConfiguredCommand(Command command, Config config, ILogger logger)
{
Command = command;
Config = config;
Logger = logger;
}
/// <summary>
/// Returns the configured value of a particular command option.
/// </summary>
public T Get<T>(Option<T> option) => option.GetValue(Config);
}
}

8
Public/Src/Tools/DropDaemon/DaemonConfig.cs → Public/Src/Tools/ServicePipDaemon/DaemonConfig.cs
Просмотреть файл

@ -5,18 +5,20 @@ using System;
using System.Diagnostics.ContractsLight;
using BuildXL.Ipc.Interfaces;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Daemon configuration.
/// </summary>
public sealed class DaemonConfig : IServerConfig, IClientConfig
{
// <nodoc />
internal ILogger Logger { get; }
/// <nodoc/>
public ILogger Logger { get; }
/// <inheritdoc/>
ILogger IServerConfig.Logger => Logger;
/// <inheritdoc/>
ILogger IClientConfig.Logger => Logger;
#region ConfigOptions

7
Public/Src/Tools/DropDaemon/DropDaemonException.cs → Public/Src/Tools/ServicePipDaemon/DaemonException.cs
Просмотреть файл

@ -2,16 +2,17 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Runtime.Remoting.Messaging;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Generic DropDaemon exception.
/// </summary>
public sealed class DropDaemonException : Exception
public sealed class DaemonException : Exception
{
/// <nodoc/>
public DropDaemonException(string message)
public DaemonException(string message)
: base(message) { }
}
}

2
Public/Src/Tools/DropDaemon/DisabledCloudBuildLogger.cs → Public/Src/Tools/ServicePipDaemon/DisabledCloudBuildLogger.cs
Просмотреть файл

@ -3,7 +3,7 @@
using BuildXL.Tracing.CloudBuild;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Implementation of <see cref="ICloudBuildLogger"/> that doesn't do anything.

2
Public/Src/Tools/DropDaemon/DropEtwListener.cs → Public/Src/Tools/ServicePipDaemon/DropEtwListener.cs
Просмотреть файл

@ -7,7 +7,7 @@ using System.Diagnostics.Tracing;
using BuildXL.Tracing;
using BuildXL.Tracing.CloudBuild;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Simple listener for Drop events.

2
Public/Src/Tools/DropDaemon/ICloudBuildLogger.cs → Public/Src/Tools/ServicePipDaemon/ICloudBuildLogger.cs
Просмотреть файл

@ -4,7 +4,7 @@
using System.Diagnostics.ContractsLight;
using BuildXL.Tracing.CloudBuild;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Interface for sending ETW events.

2
Public/Src/Tools/DropDaemon/Reloader.cs → Public/Src/Tools/ServicePipDaemon/Reloader.cs
Просмотреть файл

@ -7,7 +7,7 @@ using System.Diagnostics.ContractsLight;
using System.Linq;
using JetBrains.Annotations;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Utility class for synchronizing request to "reload" (reconstruct) some underlying value.

4
Public/Src/Tools/DropDaemon/ReloadingDropServiceClient.cs → Public/Src/Tools/ServicePipDaemon/ReloadingDropServiceClient.cs
Просмотреть файл

@ -15,10 +15,10 @@ using Microsoft.VisualStudio.Services.Drop.App.Core;
using Microsoft.VisualStudio.Services.Drop.WebApi;
using Microsoft.VisualStudio.Services.ItemStore.Common;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// <see cref="IDropServiceClient"/> which will retrie every operation in case <see cref="Microsoft.VisualStudio.Services.Common.VssUnauthorizedException"/> is caught.
/// <see cref="IDropServiceClient"/> which will retry every operation in case <see cref="Microsoft.VisualStudio.Services.Common.VssUnauthorizedException"/> is caught.
/// </summary>
public sealed class ReloadingDropServiceClient : IDropServiceClient
{

502
Public/Src/Tools/ServicePipDaemon/ServicePipDaemon.cs Normal file
Просмотреть файл

@ -0,0 +1,502 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.ContractsLight;
using System.Globalization;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BuildXL.Ipc;
using BuildXL.Ipc.Common;
using BuildXL.Ipc.ExternalApi;
using BuildXL.Ipc.Interfaces;
using BuildXL.Utilities;
using BuildXL.Utilities.CLI;
using BuildXL.Utilities.Tracing;
using JetBrains.Annotations;
using static BuildXL.Utilities.FormattableStringEx;
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Responsible for accepting and handling TCP/IP connections from clients.
/// </summary>
public abstract class ServicePipDaemon : IDisposable, IIpcOperationExecutor
{
internal static readonly IIpcProvider IpcProvider = IpcFactory.GetProvider();
internal static readonly List<Option> DaemonConfigOptions = new List<Option>();
/// <summary>Initialized commands</summary>
protected static readonly Dictionary<string, Command> Commands = new Dictionary<string, Command>();
private const string LogFileName = "ServiceDaemon";
/// <nodoc/>
public const string LogPrefix = "(SPD) ";
/// <summary>Daemon configuration.</summary>
public DaemonConfig Config { get; }
/// <summary>Task to wait on for the completion result.</summary>
public Task Completion => m_server.Completion;
/// <summary>Client for talking to BuildXL.</summary>
[CanBeNull]
public Client ApiClient { get; }
/// <nodoc />
protected readonly ICloudBuildLogger m_etwLogger;
/// <nodoc />
protected readonly IServer m_server;
/// <nodoc />
protected readonly IParser m_parser;
/// <nodoc />
protected readonly ILogger m_logger;
/// <nodoc />
protected readonly CounterCollection<DaemonCounter> m_counters = new CounterCollection<DaemonCounter>();
/// <nodoc />
protected enum DaemonCounter
{
/// <nodoc/>
[CounterType(CounterType.Stopwatch)]
ParseArgsDuration,
/// <nodoc/>
[CounterType(CounterType.Stopwatch)]
ServerActionDuration,
/// <nodoc/>
QueueDurationMs,
}
/// <nodoc />
public ILogger Logger => m_logger;
#region Options and commands
/// <nodoc />
public static readonly StrOption Moniker = RegisterDaemonConfigOption(new StrOption("moniker")
{
ShortName = "m",
HelpText = "Moniker to identify client/server communication",
});
/// <nodoc />
public static readonly IntOption MaxConcurrentClients = RegisterDaemonConfigOption(new IntOption("maxConcurrentClients")
{
HelpText = "OBSOLETE due to the hardcoded config. (Maximum number of clients to serve concurrently)",
DefaultValue = DaemonConfig.DefaultMaxConcurrentClients,
});
/// <nodoc />
public static readonly IntOption MaxConnectRetries = RegisterDaemonConfigOption(new IntOption("maxConnectRetries")
{
HelpText = "Maximum number of retries to establish a connection with a running daemon",
DefaultValue = DaemonConfig.DefaultMaxConnectRetries,
});
/// <nodoc />
public static readonly IntOption ConnectRetryDelayMillis = RegisterDaemonConfigOption(new IntOption("connectRetryDelayMillis")
{
HelpText = "Delay between consecutive retries to establish a connection with a running daemon",
DefaultValue = (int)DaemonConfig.DefaultConnectRetryDelay.TotalMilliseconds,
});
/// <nodoc />
public static readonly BoolOption ShellExecute = RegisterDaemonConfigOption(new BoolOption("shellExecute")
{
HelpText = "Use shell execute to start the daemon process (a shell window will be created and displayed)",
DefaultValue = false,
});
/// <nodoc />
public static readonly BoolOption StopOnFirstFailure = RegisterDaemonConfigOption(new BoolOption("stopOnFirstFailure")
{
HelpText = "Daemon process should terminate after first failed operation (e.g., 'drop create' fails because the drop already exists).",
DefaultValue = DaemonConfig.DefaultStopOnFirstFailure,
});
/// <nodoc />
public static readonly BoolOption EnableCloudBuildIntegration = RegisterDaemonConfigOption(new BoolOption("enableCloudBuildIntegration")
{
ShortName = "ecb",
HelpText = "Enable logging ETW events for CloudBuild to pick up",
DefaultValue = DaemonConfig.DefaultEnableCloudBuildIntegration,
});
/// <nodoc />
public static readonly BoolOption Verbose = RegisterDaemonConfigOption(new BoolOption("verbose")
{
ShortName = "v",
HelpText = "Verbose logging",
IsRequired = false,
DefaultValue = false,
});
/// <nodoc />
public static readonly StrOption LogDir = RegisterDaemonConfigOption(new StrOption("logDir")
{
ShortName = "log",
HelpText = "Log directory",
IsRequired = false
});
/// <nodoc />
public static readonly StrOption File = new StrOption("file")
{
ShortName = "f",
HelpText = "File path",
IsRequired = false,
IsMultiValue = true,
};
/// <nodoc />
public static readonly StrOption IpcServerMonikerRequired = new StrOption("ipcServerMoniker")
{
ShortName = "dm",
HelpText = "IPC moniker identifying a running BuildXL IPC server",
IsRequired = true,
};
/// <nodoc />
public static readonly StrOption HelpNoNameOption = new StrOption(string.Empty)
{
HelpText = "Command name",
};
/// <nodoc />
public static readonly StrOption IpcServerMonikerOptional = new StrOption(longName: IpcServerMonikerRequired.LongName)
{
ShortName = IpcServerMonikerRequired.ShortName,
HelpText = IpcServerMonikerRequired.HelpText,
IsRequired = false,
};
/// <nodoc />
protected static T RegisterOption<T>(List<Option> options, T option) where T : Option
{
options.Add(option);
return option;
}
/// <nodoc />
protected static T RegisterDaemonConfigOption<T>(T option) where T : Option => RegisterOption(DaemonConfigOptions, option);
/// <remarks>
/// The <see cref="DaemonConfigOptions"/> options are added to every command.
/// A non-mandatory string option "name" is added as well, which operation
/// commands may want to use to explicitly specify a particular end point name
/// (e.g., the target drop name).
/// </remarks>
public static Command RegisterCommand(
string name,
IEnumerable<Option> options = null,
ServerAction serverAction = null,
ClientAction clientAction = null,
string description = null,
bool needsIpcClient = true,
bool addDaemonConfigOptions = true)
{
var opts = (options ?? new Option[0]).ToList();
if (addDaemonConfigOptions)
{
opts.AddRange(DaemonConfigOptions);
}
if (!opts.Exists(opt => opt.LongName == "name"))
{
opts.Add(new Option(longName: "name")
{
ShortName = "n",
});
}
var cmd = new Command(name, opts, serverAction, clientAction, description, needsIpcClient);
Commands[cmd.Name] = cmd;
return cmd;
}
/// <nodoc />
protected static readonly Command HelpCmd = RegisterCommand(
name: "help",
description: "Prints a help message (usage).",
options: new[] { HelpNoNameOption },
needsIpcClient: false,
clientAction: (conf, rpc) =>
{
string cmdName = conf.Get(HelpNoNameOption);
bool cmdNotSpecified = string.IsNullOrWhiteSpace(cmdName);
if (cmdNotSpecified)
{
Console.WriteLine(Usage());
return 0;
}
Command requestedHelpForCommand;
var requestedCommandFound = Commands.TryGetValue(cmdName, out requestedHelpForCommand);
if (requestedCommandFound)
{
Console.WriteLine(requestedHelpForCommand.Usage(conf.Config.Parser));
return 0;
}
else
{
Console.WriteLine(Usage());
return 1;
}
});
/// <nodoc />
public static readonly Command StopDaemonCmd = RegisterCommand(
name: "stop",
description: "[RPC] Stops the daemon process running on specified port; fails if no such daemon is running.",
clientAction: AsyncRPCSend,
serverAction: (conf, daemon) =>
{
conf.Logger.Info("[STOP] requested");
daemon.RequestStop();
return Task.FromResult(IpcResult.Success());
});
/// <nodoc />
public static readonly Command CrashDaemonCmd = RegisterCommand(
name: "crash",
description: "[RPC] Stops the server process by crashing it.",
clientAction: AsyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[CRASH] requested");
Environment.Exit(-1);
return Task.FromResult(IpcResult.Success());
});
/// <nodoc />
public static readonly Command PingDaemonCmd = RegisterCommand(
name: "ping",
description: "[RPC] Pings the daemon process.",
clientAction: SyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[PING] received");
return Task.FromResult(IpcResult.Success("Alive!"));
});
/// <nodoc />
public static readonly Command TestReadFile = RegisterCommand(
name: "test-readfile",
description: "[RPC] Sends a request to the daemon to read a file.",
options: new Option[] { File },
clientAction: SyncRPCSend,
serverAction: (conf, daemon) =>
{
daemon.Logger.Info("[READFILE] received");
var result = IpcResult.Success(System.IO.File.ReadAllText(conf.Get(File)));
daemon.Logger.Info("[READFILE] succeeded");
return Task.FromResult(result);
});
#endregion
/// <nodoc />
public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, ILogger logger, IIpcProvider rpcProvider = null, Client client = null)
{
Contract.Requires(daemonConfig != null);
Config = daemonConfig;
m_parser = parser;
ApiClient = client;
m_logger = logger;
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
m_server = rpcProvider.GetServer(Config.Moniker, Config);
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
}
/// <summary>
/// Starts to listen for client connections. As soon as a connection is received,
/// it is placed in an action block from which it is picked up and handled asynchronously
/// (in the <see cref="ParseAndExecuteCommand"/> method).
/// </summary>
public void Start()
{
m_server.Start(this);
}
/// <summary>
/// Requests shut down, causing this daemon to immediately stop listening for TCP/IP
/// connections. Any pending requests, however, will be processed to completion.
/// </summary>
public void RequestStop()
{
m_server.RequestStop();
}
/// <summary>
/// Calls <see cref="RequestStop"/> then waits for <see cref="Completion"/>.
/// </summary>
public Task RequestStopAndWaitForCompletionAsync()
{
RequestStop();
return Completion;
}
/// <inheritdoc />
public void Dispose()
{
m_server.Dispose();
ApiClient?.Dispose();
m_logger.Dispose();
}
private async Task<IIpcResult> ParseAndExecuteCommand(IIpcOperation operation)
{
string cmdLine = operation.Payload;
m_logger.Verbose("Command received: {0}", cmdLine);
ConfiguredCommand conf;
using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration))
{
conf = ParseArgs(cmdLine, m_parser);
}
IIpcResult result;
using (var duration = m_counters.StartStopwatch(DaemonCounter.ServerActionDuration))
{
result = await conf.Command.ServerAction(conf, this);
result.ActionDuration = duration.Elapsed;
}
TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime;
m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds);
return result;
}
Task<IIpcResult> IIpcOperationExecutor.ExecuteAsync(IIpcOperation operation)
{
Contract.Requires(operation != null);
return ParseAndExecuteCommand(operation);
}
/// <summary>
/// Parses a string and returns a ConfiguredCommand.
/// </summary>
public static ConfiguredCommand ParseArgs(string allArgs, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
{
return ParseArgs(parser.SplitArgs(allArgs), parser, logger, ignoreInvalidOptions);
}
/// <summary>
/// Parses a list of arguments and returns a ConfiguredCommand.
/// </summary>
public static ConfiguredCommand ParseArgs(string[] args, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
{
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
if (args.Length == 0)
{
throw new ArgumentException(I($"Command is required. {usageMessage.Value}"));
}
var argsQueue = new Queue<string>(args);
string cmdName = argsQueue.Dequeue();
if (!Commands.TryGetValue(cmdName, out Command cmd))
{
throw new ArgumentException(I($"No command '{cmdName}' is found. {usageMessage.Value}"));
}
var sw = Stopwatch.StartNew();
Config conf = BuildXL.Utilities.CLI.Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
var parseTime = sw.Elapsed;
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), ServicePipDaemon.LogPrefix);
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
return new ConfiguredCommand(cmd, conf, logger);
}
/// <summary>
/// Creates DaemonConfig using the values specified on the ConfiguredCommand
/// </summary>
public static DaemonConfig CreateDaemonConfig(ConfiguredCommand conf)
{
return new DaemonConfig(
logger: conf.Logger,
moniker: conf.Get(Moniker),
maxConnectRetries: conf.Get(MaxConnectRetries),
connectRetryDelay: TimeSpan.FromMilliseconds(conf.Get(ConnectRetryDelayMillis)),
stopOnFirstFailure: conf.Get(StopOnFirstFailure),
enableCloudBuildIntegration: conf.Get(EnableCloudBuildIntegration));
}
/// <summary>
/// Creates anIPC client using the config from a ConfiguredCommand
/// </summary>
public static IClient CreateClient(ConfiguredCommand conf)
{
var daemonConfig = CreateDaemonConfig(conf);
return IpcProvider.GetClient(daemonConfig.Moniker, daemonConfig);
}
private static string Usage()
{
var builder = new StringBuilder();
var len = Commands.Keys.Max(cmdName => cmdName.Length);
foreach (var cmd in Commands.Values)
{
builder.AppendLine(string.Format(CultureInfo.InvariantCulture, " {0,-" + len + "} : {1}", cmd.Name, cmd.Description));
}
return builder.ToString();
}
/// <nodoc />
protected static int SyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, true);
/// <nodoc />
protected static int AsyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, false);
/// <nodoc />
protected static int RPCSend(ConfiguredCommand conf, IClient rpc, bool isSync)
{
var rpcResult = RPCSendCore(conf, rpc, isSync);
conf.Logger.Info(
"Command '{0}' {1} (exit code: {2}). {3}",
conf.Command.Name,
rpcResult.Succeeded ? "succeeded" : "failed",
(int)rpcResult.ExitCode,
rpcResult.Payload);
return (int)rpcResult.ExitCode;
}
private static IIpcResult RPCSendCore(ConfiguredCommand conf, IClient rpc, bool isSync)
{
string operationPayload = ToPayload(conf);
var operation = new IpcOperation(operationPayload, waitForServerAck: isSync);
return rpc.Send(operation).GetAwaiter().GetResult();
}
/// <summary>
/// Reconstructs a full command line from a command name (<paramref name="commandName"/>)
/// and a configuration (<paramref name="config"/>).
/// </summary>
internal static string ToPayload(string commandName, Config config)
{
return commandName + " " + config.Render();
}
/// <summary>
/// Reconstructs a full command line corresponding to a <see cref="ConfiguredCommand"/>.
/// </summary>
private static string ToPayload(ConfiguredCommand cmd) => ToPayload(cmd.Command.Name, cmd.Config);
}
}

20
Public/Src/Tools/DropDaemon/Statics.cs → Public/Src/Tools/ServicePipDaemon/Statics.cs
Просмотреть файл

@ -2,36 +2,24 @@
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Diagnostics.ContractsLight;
using System.Globalization;
using BuildXL.Native.IO;
namespace Tool.DropDaemon
namespace Tool.ServicePipDaemon
{
/// <summary>
/// Various helper method, typically to be imported with "using static".
/// </summary>
public static class Statics
{
/// <summary>
/// String format with <see cref="CultureInfo.InvariantCulture"/>.
/// </summary>
public static string Inv(string format, params object[] args)
{
Contract.Requires(format != null);
return string.Format(CultureInfo.InvariantCulture, format, args);
}
/// <summary>
/// Logs an error as a line of text. Currently prints out to <code>Console.Error</code>.
/// to use whatever other
/// </summary>
public static void Error(string format, params object[] args)
public static void Error(string error)
{
if (format != null)
if (error != null)
{
Console.Error.WriteLine(Inv(format, args));
Console.Error.WriteLine(error);
}
}

49
Public/Src/Tools/ServicePipDaemon/Tool.ServicePipDaemon.dsc Normal file
Просмотреть файл

@ -0,0 +1,49 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
import {Artifact, Cmd, Transformer} from "Sdk.Transformers";
import * as BuildXLSdk from "Sdk.BuildXL";
import * as Managed from "Sdk.Managed";
import * as Deployment from "Sdk.Deployment";
import { NetFx } from "Sdk.BuildXL";
namespace ServicePipDaemon {
@@public
export const dll = !BuildXLSdk.isDaemonToolingEnabled ? undefined : BuildXLSdk.library({
assemblyName: "Tool.ServicePipDaemon",
rootNamespace: "Tool.ServicePipDaemon",
sources: globR(d`.`, "*.cs"),
references:[
importFrom("BuildXL.Engine").Scheduler.dll,
importFrom("BuildXL.Utilities.Instrumentation").Common.dll,
importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll,
importFrom("BuildXL.Utilities").dll,
importFrom("BuildXL.Utilities").Ipc.dll,
importFrom("BuildXL.Utilities").Native.dll,
importFrom("BuildXL.Utilities").Storage.dll,
importFrom("ArtifactServices.App.Shared").pkg,
importFrom("ArtifactServices.App.Shared.Cache").pkg,
importFrom("Drop.App.Core").pkg,
importFrom("Drop.Client").pkg,
importFrom("Drop.RemotableClient.Interfaces").pkg,
importFrom("ItemStore.Shared").pkg,
importFrom("Microsoft.ApplicationInsights").pkg,
importFrom("Microsoft.AspNet.WebApi.Client").pkg,
importFrom("Microsoft.Diagnostics.Tracing.TraceEvent").pkg,
importFrom("Microsoft.IdentityModel.Clients.ActiveDirectory").pkg,
...BuildXLSdk.visualStudioServicesArtifactServicesSharedPkg,
importFrom("Microsoft.VisualStudio.Services.BlobStore.Client").pkg,
importFrom("Microsoft.VisualStudio.Services.Client").pkg,
importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
importFrom("Newtonsoft.Json").pkg,
importFrom("WindowsAzure.Storage").pkg,
],
internalsVisibleTo: [
"Test.Tool.DropDaemon",
]
});
}

37
Public/Src/Tools/UnitTests/DropDaemon/CLITests.cs
Просмотреть файл

@ -11,9 +11,11 @@ using BuildXL.Ipc.Interfaces;
using BuildXL.Utilities.CLI;
using Test.BuildXL.TestUtilities.Xunit;
using Tool.DropDaemon;
using Tool.ServicePipDaemon;
using Xunit;
using Xunit.Abstractions;
using static Tool.DropDaemon.Program;
using static Tool.DropDaemon.DropDaemon;
using static Tool.ServicePipDaemon.ServicePipDaemon;
namespace Test.Tool.DropDaemon
{
@ -69,18 +71,18 @@ namespace Test.Tool.DropDaemon
public static IEnumerable<object[]> ValidCommandLinesTestData()
{
yield return new object[] { "start", new Option[] { DropName, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
yield return new object[] { "start", new Option[] { DropName, DropEndpoint, EnableCloudBuildIntegration }, new[] { "mydrop", "http://xyz", "True" } };
yield return new object[] { "create", new Option[] { DropName, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
yield return new object[] { "addfile", new[] { File, DropName }, new[] { @"""c:\x\y.txt""", "mydrop" } };
yield return new object[] { "start", new Option[] { DropNameOption, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
yield return new object[] { "start", new Option[] { DropNameOption, DropEndpoint, EnableCloudBuildIntegration }, new[] { "mydrop", "http://xyz", "True" } };
yield return new object[] { "create", new Option[] { DropNameOption, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
yield return new object[] { "addfile", new[] { File, DropNameOption }, new[] { @"""c:\x\y.txt""", "mydrop" } };
yield return new object[] { "addfile", new[] { File }, new[] { @"""c:\x\y.txt""" } };
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", "a/b/c.txt" } };
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", @"a\\b\\c.txt" } };
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", "\"a\\b\\c.txt\"" } };
yield return new object[] { "finalize", new Option[0], new string[0] };
yield return new object[] { "finalize", new[] { DropName }, new[] { "mydrop" } };
yield return new object[] { "finalize", new[] { DropNameOption }, new[] { "mydrop" } };
yield return new object[] { "stop", new Option[0], new string[0] };
yield return new object[] { "stop", new[] { DropName }, new[] { "mydrop" } };
yield return new object[] { "stop", new[] { DropNameOption }, new[] { "mydrop" } };
yield return new object[] { "addartifacts", new[] { IpcServerMonikerRequired, Directory, DirectoryId, RelativeDirectoryDropPath }, new[] { "moniker_string", @"c:\dir", "123:1:12345", "/remote/" } };
yield return new object[] { "addartifacts", new[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath }, new[] { "moniker_string", @"c:\dir\f.txt", "id1", "123:1:12345", "/remote/f.txt" } };
}
@ -109,11 +111,11 @@ namespace Test.Tool.DropDaemon
public static IEnumerable<object[]> OptionOverrideTestData()
{
yield return new object[] { DropName, "a", "b", "c" };
yield return new object[] { DropName, "a", "b", null };
yield return new object[] { DropName, "a", null, "c" };
yield return new object[] { DropName, null, "b", "c" };
yield return new object[] { DropName, null, "b", null };
yield return new object[] { DropNameOption, "a", "b", "c" };
yield return new object[] { DropNameOption, "a", "b", null };
yield return new object[] { DropNameOption, "a", null, "c" };
yield return new object[] { DropNameOption, null, "b", "c" };
yield return new object[] { DropNameOption, null, "b", null };
}
/// <summary>
@ -133,7 +135,8 @@ namespace Test.Tool.DropDaemon
public static IEnumerable<object[]> ParseArgsRequiredOptionMissingTestData()
{
foreach (var cmd in Commands.Values)
// TODO: this only iterates over DropDaemon & ServicePipDaemon commands => add symbols commands
foreach (var cmd in SupportedCommands)
{
var requiredSwitches = cmd.Options.Where(opt => opt.IsRequired).Select(opt => opt.LongName).ToList();
if (!requiredSwitches.Any())
@ -169,8 +172,8 @@ namespace Test.Tool.DropDaemon
// get either DaemonConfig or DropConfig object from the parsed command line, based on given 'property'
object configObject =
property.DeclaringType == typeof(DaemonConfig) ? (object)CreateDaemonConfig(cliConfig) :
property.DeclaringType == typeof(DropConfig) ? (object)CreateDropConfig(cliConfig) :
property.DeclaringType == typeof(DaemonConfig) ? (object)ServicePipDaemon.CreateDaemonConfig(cliConfig) :
property.DeclaringType == typeof(DropConfig) ? (object)global::Tool.DropDaemon.DropDaemon.CreateDropConfig(cliConfig) :
null;
// assert the value of that property is what we expect.
@ -204,7 +207,7 @@ namespace Test.Tool.DropDaemon
var cmd = StartCmd;
var cmdline = cmd.Name + " --name 123 --service http://xyz "; // prepend required flags in all cases
foreach (var prop in confType.GetProperties().Where(p => !p.Name.StartsWith("Default") && !p.Name.StartsWith("MaxConcurrentClients")))
foreach (var prop in confType.GetProperties().Where(p => !p.Name.StartsWith("Default") && !p.Name.StartsWith("MaxConcurrentClients") && !p.Name.StartsWith("Logger")))
{
// For current 'prop', try to find a corresponding option ('opt') in 'cmd.Options' and pick a
// representative value (according to its type) include in the command line.
@ -327,7 +330,7 @@ namespace Test.Tool.DropDaemon
private ConfiguredCommand ParseArgs(string fullCmdLine, bool ignoreInvalidOptions = false)
{
var logger = new LambdaLogger((level, format, args) => Output.WriteLine(format, args));
return Program.ParseArgs(fullCmdLine, UnixParser.Instance, logger, ignoreInvalidOptions: ignoreInvalidOptions);
return global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(fullCmdLine, UnixParser.Instance, logger, ignoreInvalidOptions: ignoreInvalidOptions);
}
}
}

12
Public/Src/Tools/UnitTests/DropDaemon/ConcurrencyTest.cs
Просмотреть файл

@ -11,9 +11,11 @@ using BuildXL.Ipc.Common.Multiplexing;
using BuildXL.Ipc.Interfaces;
using BuildXL.Utilities.CLI;
using Test.BuildXL.TestUtilities.Xunit;
using Tool.DropDaemon;
using Tool.ServicePipDaemon;
using Xunit;
using Xunit.Abstractions;
using static Tool.DropDaemon.DropDaemon;
using static Tool.ServicePipDaemon.ServicePipDaemon;
namespace Test.Tool.DropDaemon
{
@ -42,18 +44,18 @@ namespace Test.Tool.DropDaemon
.ToList();
var serverThreads = ipcMonikers
.Select(moniker => CreateThreadForCommand($"{Program.StartNoDropCmd.Name} --{Program.Moniker.LongName} " + moniker, null))
.Select(moniker => CreateThreadForCommand($"{StartNoDropCmd.Name} --{Moniker.LongName} " + moniker, null))
.ToList();
Start(serverThreads);
Thread.Sleep(100);
var clientThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, numRequestsPerService, $"{Program.PingDaemonCmd.Name} --{Program.Moniker.LongName} <moniker>");
var clientThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, numRequestsPerService, $"{PingDaemonCmd.Name} --{Moniker.LongName} <moniker>");
Start(clientThreads);
Join(clientThreads);
var serverShutdownThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, 1, $"{Program.StopDaemonCmd.Name} --{Program.Moniker.LongName} <moniker>");
var serverShutdownThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, 1, $"{StopDaemonCmd.Name} --{Moniker.LongName} <moniker>");
Start(serverShutdownThreads);
Join(serverShutdownThreads);
@ -98,7 +100,7 @@ namespace Test.Tool.DropDaemon
Console.WriteLine(format);
Output.WriteLine(format);
});
ConfiguredCommand conf = Program.ParseArgs(cmdLine, UnixParser.Instance, logger);
ConfiguredCommand conf = ParseArgs(cmdLine, UnixParser.Instance, logger);
var exitCode = conf.Command.ClientAction(conf, client);
Assert.Equal(0, exitCode);
});

25
Public/Src/Tools/UnitTests/DropDaemon/DropOperationTests.cs
Просмотреть файл

@ -21,6 +21,7 @@ using Microsoft.VisualStudio.Services.Common;
using Microsoft.VisualStudio.Services.Drop.WebApi;
using Test.BuildXL.TestUtilities.Xunit;
using Tool.DropDaemon;
using Tool.ServicePipDaemon;
using Xunit;
using Xunit.Abstractions;
@ -184,7 +185,7 @@ namespace Test.Tool.DropDaemon
XAssert.IsTrue(result.Succeeded);
// calling MaterializeFile fails because no BuildXL server is running
Assert.Throws<DropDaemonException>(() => addFileItem.EnsureMaterialized().GetAwaiter().GetResult());
Assert.Throws<DaemonException>(() => addFileItem.EnsureMaterialized().GetAwaiter().GetResult());
});
}
@ -259,7 +260,7 @@ namespace Test.Tool.DropDaemon
var dropClient = new MockDropClient(addFileFunc: (item) =>
{
Assert.NotNull(item.BlobIdentifier);
var ex = Assert.Throws<DropDaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
var ex = Assert.Throws<DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
// rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it)
throw ex;
@ -311,7 +312,7 @@ namespace Test.Tool.DropDaemon
{
var ipcResult = await daemon.AddFileAsync(new DropItemForFile(targetFile), symlinkTester: (file) => file == targetFile ? true : false);
Assert.False(ipcResult.Succeeded, "adding symlink to drop succeeded while it was expected to fail");
Assert.True(ipcResult.Payload.Contains(Daemon.SymlinkAddErrorMessagePrefix));
Assert.True(ipcResult.Payload.Contains(global::Tool.DropDaemon.DropDaemon.SymlinkAddErrorMessagePrefix));
});
}
@ -382,7 +383,9 @@ namespace Test.Tool.DropDaemon
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}", new UnixParser());
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
@ -415,7 +418,9 @@ namespace Test.Tool.DropDaemon
{
// only hash and file rewrite count are important here; the rest are just fake values
var hash = FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.CreateSpecialValue(1)).Render();
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {daemon.Config.Moniker} --file non-existent-file.txt --dropPath remote-file-name.txt --hash {hash} --fileId 12345:{(isSourceFile ? 0 : 1)}", new UnixParser());
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {daemon.Config.Moniker} --file non-existent-file.txt --dropPath remote-file-name.txt --hash {hash} --fileId 12345:{(isSourceFile ? 0 : 1)}",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
@ -473,7 +478,9 @@ namespace Test.Tool.DropDaemon
dropClient,
(daemon, etwListener) =>
{
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser());
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*",
new UnixParser());
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
XAssert.IsTrue(dropPaths.Count == 0);
@ -532,13 +539,13 @@ namespace Test.Tool.DropDaemon
Assert.Equal(shouldSucceed, rpcResult.Succeeded);
}
private void WithSetup(IDropClient dropClient, Action<Daemon, DropEtwListener> action, Client apiClient = null)
private void WithSetup(IDropClient dropClient, Action<global::Tool.DropDaemon.DropDaemon, DropEtwListener> action, Client apiClient = null)
{
var etwListener = ConfigureEtwLogging();
string moniker = Program.IpcProvider.RenderConnectionString(Program.IpcProvider.CreateNewMoniker());
string moniker = ServicePipDaemon.IpcProvider.RenderConnectionString(ServicePipDaemon.IpcProvider.CreateNewMoniker());
var daemonConfig = new DaemonConfig(VoidLogger.Instance, moniker: moniker, enableCloudBuildIntegration: false);
var dropConfig = new DropConfig(string.Empty, null);
var daemon = new Daemon(UnixParser.Instance, daemonConfig, dropConfig, Task.FromResult(dropClient), client: apiClient);
var daemon = new global::Tool.DropDaemon.DropDaemon(UnixParser.Instance, daemonConfig, dropConfig, Task.FromResult(dropClient), client: apiClient);
action(daemon, etwListener);
}

2
Public/Src/Tools/UnitTests/DropDaemon/ReloaderTests.cs
Просмотреть файл

@ -5,7 +5,7 @@ using System;
using System.Linq;
using System.Threading;
using Test.BuildXL.TestUtilities.Xunit;
using Tool.DropDaemon;
using Tool.ServicePipDaemon;
using Xunit;
using Xunit.Abstractions;

2
Public/Src/Tools/UnitTests/DropDaemon/ReloadingDropServiceClientTest.cs
Просмотреть файл

@ -15,7 +15,7 @@ using Microsoft.VisualStudio.Services.Drop.App.Core;
using Microsoft.VisualStudio.Services.Drop.WebApi;
using Microsoft.VisualStudio.Services.ItemStore.Common;
using Test.BuildXL.TestUtilities.Xunit;
using Tool.DropDaemon;
using Tool.ServicePipDaemon;
using Xunit;
using Xunit.Abstractions;

1
Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc
Просмотреть файл

@ -12,6 +12,7 @@ namespace Test.Tool.DropDaemon {
importFrom("BuildXL.Cache.ContentStore").UtilitiesCore.dll,
importFrom("BuildXL.Engine").Scheduler.dll,
importFrom("BuildXL.Tools.DropDaemon").exe,
importFrom("BuildXL.Tools").ServicePipDaemon.dll,
importFrom("BuildXL.Utilities").dll,
importFrom("BuildXL.Utilities").Ipc.dll,
importFrom("BuildXL.Utilities").Storage.dll,

20
Public/Src/Utilities/Ipc/Common/IpcResult.cs
Просмотреть файл

@ -50,23 +50,32 @@ namespace BuildXL.Ipc.Common
/// <nodoc />
public IpcResultTimestamp Timestamp { get; }
/// <inheritdoc/>
public TimeSpan ActionDuration { get; set; }
/// <summary>
/// Creates a result representing a successful IPC operation.
/// </summary>
public static IIpcResult Success(string payload = null) => new IpcResult(IpcResultStatus.Success, payload);
/// <nodoc />
public IpcResult(IpcResultStatus status, string payload)
public IpcResult(IpcResultStatus status, string payload) : this(status, payload, TimeSpan.Zero)
{
}
/// <nodoc />
public IpcResult(IpcResultStatus status, string payload, TimeSpan actionDuraion)
{
m_exitCode = status;
m_payload = payload ?? string.Empty;
Timestamp = new IpcResultTimestamp();
ActionDuration = actionDuraion;
}
/// <inheritdoc/>
public override string ToString()
{
return I($"{{succeeded: {Succeeded}, payload: '{Payload}'}}");
return I($"{{succeeded: {Succeeded}, payload: '{Payload}', ActionDuration: {ActionDuration}}}");
}
/// <summary>
@ -90,6 +99,7 @@ namespace BuildXL.Ipc.Common
{
await Utils.WriteByteAsync(stream, (byte)result.ExitCode, token);
await Utils.WriteStringAsync(stream, result.Payload, token);
await Utils.WriteLongAsync(stream, result.ActionDuration.Ticks, token);
await stream.FlushAsync(token);
}
@ -104,7 +114,9 @@ namespace BuildXL.Ipc.Common
byte statusByte = await Utils.ReadByteAsync(stream, token);
Utils.CheckSerializationFormat(Enum.IsDefined(typeof(IpcResultStatus), statusByte), "unknown IpcResult.Status byte: {0}", statusByte);
string payload = await Utils.ReadStringAsync(stream, token);
return new IpcResult((IpcResultStatus)statusByte, payload);
long actionDuration = await Utils.ReadLongAsync(stream, token);
return new IpcResult((IpcResultStatus)statusByte, payload, TimeSpan.FromTicks(actionDuration));
}
/// <summary>
@ -120,7 +132,7 @@ namespace BuildXL.Ipc.Common
lhs.Succeeded && !rhs.Succeeded ? rhs.ExitCode :
!lhs.Succeeded && rhs.Succeeded ? lhs.ExitCode :
IpcResultStatus.GenericError;
return new IpcResult(mergedStatus, lhs.Payload + Environment.NewLine + rhs.Payload);
return new IpcResult(mergedStatus, lhs.Payload + Environment.NewLine + rhs.Payload, lhs.ActionDuration + rhs.ActionDuration);
}
/// <summary>

6
Public/Src/Utilities/Ipc/Interfaces/IIpcResult.cs
Просмотреть файл

@ -1,6 +1,7 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
using System;
using System.Diagnostics.ContractsLight;
using BuildXL.Ipc.Common;
@ -71,5 +72,10 @@ namespace BuildXL.Ipc.Interfaces
/// <nodoc/>
IpcResultTimestamp Timestamp { get; }
/// <summary>
/// (Optional) Duration of the action executed by a server.
/// </summary>
TimeSpan ActionDuration { get; set; }
}
}