зеркало из https://github.com/microsoft/BuildXL.git
Родитель
f6e1cab039
Коммит
5d2cfeb8b3
|
@ -8,7 +8,7 @@ if NOT DEFINED ENLISTMENTROOT (
|
|||
echo =======================================================
|
||||
echo Building BuildXL
|
||||
echo =======================================================
|
||||
call %ENLISTMENTROOT%\bxl.cmd -deploy dev /server- /f:output='Out\Bin\debug\net472\*'
|
||||
call %ENLISTMENTROOT%\bxl.cmd -deploydev /server-
|
||||
if %ERRORLEVEL% NEQ 0 (
|
||||
echo.
|
||||
echo ERROR: BuildXL build failed.
|
||||
|
@ -16,7 +16,7 @@ if %ERRORLEVEL% NEQ 0 (
|
|||
endlocal && exit /b 1
|
||||
)
|
||||
|
||||
set BUILDXL_BIN_DIRECTORY=%ENLISTMENTROOT%\Out\Bin\Debug\net472
|
||||
set BUILDXL_BIN_DIRECTORY=%ENLISTMENTROOT%\Out\Bin\Debug\win-x64
|
||||
|
||||
if NOT DEFINED TF_ROLLING_DROPNAME (
|
||||
set TF_ROLLING_DROPNAME=%USERNAME%-%random%
|
||||
|
|
|
@ -115,11 +115,17 @@ export const restrictTestRunToDebugNet461OnWindows =
|
|||
(qualifier.targetFramework !== "netcoreapp3.0" && qualifier.targetFramework !== "net472") ||
|
||||
(Context.isWindowsOS() && qualifier.targetRuntime === "osx-x64");
|
||||
|
||||
@@public
|
||||
/***
|
||||
* Whether service pip daemon tooling is included with the BuildXL deployment
|
||||
*/
|
||||
export const isDaemonToolingEnabled = Flags.isMicrosoftInternal && isFullFramework;
|
||||
|
||||
/***
|
||||
* Whether drop tooling is included with the BuildXL deployment
|
||||
*/
|
||||
@@public
|
||||
export const isDropToolingEnabled = Flags.isMicrosoftInternal && isFullFramework;
|
||||
export const isDropToolingEnabled = isDaemonToolingEnabled && Flags.isMicrosoftInternal && isFullFramework;
|
||||
|
||||
namespace Flags {
|
||||
export declare const qualifier: {};
|
||||
|
|
|
@ -1,131 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Utilities.CLI;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
internal delegate int ClientAction(ConfiguredCommand conf, IClient rpc);
|
||||
|
||||
internal delegate Task<IIpcResult> ServerAction(ConfiguredCommand conf, Daemon daemon);
|
||||
|
||||
/// <summary>
|
||||
/// A command has a name, description, a list of options it supports, and two actions:
|
||||
/// one for executing this command on the client, and one for executing it on the server.
|
||||
///
|
||||
/// When this program (DropDaemon.exe) is invoked, command line arguments are parsed to
|
||||
/// determine the command specified by the user. That command is then interpreted by
|
||||
/// executing its <see cref="ClientAction"/>. Most of DropDaemon's commands will be
|
||||
/// RPC calls, i.e., when a command is received via the command line, it is to be
|
||||
/// marshaled and sent over to a running DropDaemon server via an RPC. In such a case,
|
||||
/// the client action simply invokes <see cref="IClient.Send"/>. When an RPC is
|
||||
/// received by a DropDaemon server (<see cref="Daemon.ParseAndExecuteCommand"/>), a
|
||||
/// <see cref="Command"/> is unmarshaled from the payload of the RPC operation and the
|
||||
/// command is interpreted on the server by executing its <see cref="ServerAction"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Immutable.
|
||||
/// </remarks>
|
||||
internal sealed class Command
|
||||
{
|
||||
/// <summary>A unique command name.</summary>
|
||||
internal string Name { get; }
|
||||
|
||||
/// <summary>Arbitrary description.</summary>
|
||||
internal string Description { get; }
|
||||
|
||||
/// <summary>Options that may/must be passed to this command.</summary>
|
||||
internal IReadOnlyCollection<Option> Options { get; }
|
||||
|
||||
/// <summary>Action to be executed when this command is received via the command line.</summary>
|
||||
internal ClientAction ClientAction { get; }
|
||||
|
||||
/// <summary>Action to be executed when this command is received via an RPC call.</summary>
|
||||
internal ServerAction ServerAction { get; }
|
||||
|
||||
/// <summary>Whether this command requires an IpcClient; defaults to true.</summary>
|
||||
internal bool NeedsIpcClient { get; }
|
||||
|
||||
/// <nodoc />
|
||||
internal Command(
|
||||
string name,
|
||||
IEnumerable<Option> options = null,
|
||||
ServerAction serverAction = null,
|
||||
ClientAction clientAction = null,
|
||||
string description = null,
|
||||
bool needsIpcClient = true)
|
||||
{
|
||||
Contract.Requires(name != null);
|
||||
|
||||
Name = name;
|
||||
ServerAction = serverAction;
|
||||
ClientAction = clientAction;
|
||||
Description = description;
|
||||
Options = options.ToList();
|
||||
NeedsIpcClient = needsIpcClient;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a functional composition of a number of <see cref="ServerAction"/> functions,
|
||||
/// where the results are merged by calling <see cref="IpcResult.Merge(IIpcResult, IIpcResult)"/>.
|
||||
/// </summary>
|
||||
internal static ServerAction Compose(params ServerAction[] actions)
|
||||
{
|
||||
Contract.Requires(actions != null);
|
||||
Contract.Requires(actions.Length > 0);
|
||||
|
||||
var first = actions.First();
|
||||
return actions.Skip(1).Aggregate(first, (accumulator, currentAction) => new ServerAction(async (conf, daemon) =>
|
||||
{
|
||||
var lhsResult = await accumulator(conf, daemon);
|
||||
var rhsResult = await currentAction(conf, daemon);
|
||||
return IpcResult.Merge(lhsResult, rhsResult);
|
||||
}));
|
||||
}
|
||||
|
||||
internal string Usage(IParser parser)
|
||||
{
|
||||
var result = new StringBuilder();
|
||||
var tab = " ";
|
||||
result.AppendLine("NAME");
|
||||
result.Append(tab).Append(Name).Append(" - ").AppendLine(Description);
|
||||
result.AppendLine();
|
||||
result.AppendLine("SWITCHES");
|
||||
var optsSorted = Options
|
||||
.OrderBy(o => o.IsRequired ? 0 : 1)
|
||||
.ThenBy(o => o.LongName)
|
||||
.ToArray();
|
||||
result.AppendLine(parser.Usage(optsSorted, tab, tab));
|
||||
return result.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple wrapper class that holds a <see cref="Command"/> and a <see cref="Config"/>
|
||||
/// containing actual values for the command's <see cref="Command.Options"/>.
|
||||
/// </summary>
|
||||
internal sealed class ConfiguredCommand
|
||||
{
|
||||
internal Command Command { get; }
|
||||
|
||||
internal Config Config { get; }
|
||||
|
||||
internal ILogger Logger { get; }
|
||||
|
||||
internal ConfiguredCommand(Command command, Config config, ILogger logger)
|
||||
{
|
||||
Command = command;
|
||||
Config = config;
|
||||
Logger = logger;
|
||||
}
|
||||
|
||||
internal T Get<T>(Option<T> option) => option.GetValue(Config);
|
||||
}
|
||||
}
|
|
@ -1,438 +0,0 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.ExternalApi;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Tracing.CloudBuild;
|
||||
using BuildXL.Utilities.CLI;
|
||||
using BuildXL.Utilities.Tracing;
|
||||
using JetBrains.Annotations;
|
||||
using Microsoft.VisualStudio.Services.Common;
|
||||
using Microsoft.VisualStudio.Services.Drop.WebApi;
|
||||
using Newtonsoft.Json;
|
||||
using static Tool.DropDaemon.Statics;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Responsible for accepting and handling TCP/IP connections from clients.
|
||||
/// </summary>
|
||||
public sealed class Daemon : IDisposable, IIpcOperationExecutor
|
||||
{
|
||||
/// <summary>Prefix for the error message of the exception that gets thrown when a symlink is attempted to be added to drop.</summary>
|
||||
internal const string SymlinkAddErrorMessagePrefix = "SymLinks may not be added to drop: ";
|
||||
|
||||
private const string LogFileName = "DropDaemon";
|
||||
|
||||
/// <nodoc/>
|
||||
public const string DropDLogPrefix = "(DropD) ";
|
||||
|
||||
/// <summary>Daemon configuration.</summary>
|
||||
public DaemonConfig Config { get; }
|
||||
|
||||
/// <summary>Drop configuration.</summary>
|
||||
public DropConfig DropConfig { get; }
|
||||
|
||||
/// <summary>Task to wait on for the completion result.</summary>
|
||||
public Task Completion => m_server.Completion;
|
||||
|
||||
/// <summary>Name of the drop this daemon is constructing.</summary>
|
||||
public string DropName => DropConfig.Name;
|
||||
|
||||
/// <summary>Client for talking to BuildXL.</summary>
|
||||
[CanBeNull]
|
||||
public Client ApiClient { get; }
|
||||
|
||||
private readonly Task<IDropClient> m_dropClientTask;
|
||||
private readonly ICloudBuildLogger m_etwLogger;
|
||||
private readonly IServer m_server;
|
||||
private readonly IParser m_parser;
|
||||
|
||||
private readonly ILogger m_logger;
|
||||
|
||||
/// <nodoc />
|
||||
public ILogger Logger => m_logger;
|
||||
|
||||
/// <nodoc />
|
||||
public Daemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task<IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
|
||||
{
|
||||
Contract.Requires(daemonConfig != null);
|
||||
Contract.Requires(dropConfig != null);
|
||||
|
||||
Config = daemonConfig;
|
||||
DropConfig = dropConfig;
|
||||
m_parser = parser;
|
||||
ApiClient = client;
|
||||
m_logger = !string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, Config.Moniker, dropConfig.Verbose, DropDLogPrefix) : Config.Logger;
|
||||
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
|
||||
|
||||
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
|
||||
m_server = rpcProvider.GetServer(Config.Moniker, Config);
|
||||
|
||||
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
|
||||
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient)new VsoClient(m_logger, dropConfig));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts to listen for client connections. As soon as a connection is received,
|
||||
/// it is placed in an action block from which it is picked up and handled asynchronously
|
||||
/// (in the <see cref="ParseAndExecuteCommand"/> method).
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
m_server.Start(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Requests shut down, causing this daemon to immediatelly stop listening for TCP/IP
|
||||
/// connections. Any pending requests, however, will be processed to completion.
|
||||
/// </summary>
|
||||
public void RequestStop()
|
||||
{
|
||||
m_server.RequestStop();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calls <see cref="RequestStop"/> then waits for <see cref="Completion"/>.
|
||||
/// </summary>
|
||||
public Task RequestStopAndWaitForCompletionAsync()
|
||||
{
|
||||
RequestStop();
|
||||
return Completion;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous version of <see cref="CreateAsync"/>
|
||||
/// </summary>
|
||||
public IIpcResult Create()
|
||||
{
|
||||
return CreateAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
|
||||
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
|
||||
/// result of this operation.
|
||||
/// </summary>
|
||||
public async Task<IIpcResult> CreateAsync()
|
||||
{
|
||||
DropCreationEvent dropCreationEvent =
|
||||
await SendDropEtwEvent(
|
||||
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
|
||||
|
||||
return dropCreationEvent.Succeeded
|
||||
? IpcResult.Success(Inv("Drop {0} created.", DropName))
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, dropCreationEvent.ErrorMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop addfile' operation by delegating to <see cref="IDropClient.AddFileAsync"/>.
|
||||
/// Handles drop-related exceptions by omitting their stack traces.
|
||||
/// </summary>
|
||||
public Task<IIpcResult> AddFileAsync(IDropItem dropItem)
|
||||
{
|
||||
return AddFileAsync(dropItem, IsSymLinkOrMountPoint);
|
||||
}
|
||||
|
||||
internal async Task<IIpcResult> AddFileAsync(IDropItem dropItem, Func<string, bool> symlinkTester)
|
||||
{
|
||||
Contract.Requires(dropItem != null);
|
||||
|
||||
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
|
||||
if (File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath);
|
||||
}
|
||||
|
||||
return await WrapDropErrorsIntoIpcResult(async () =>
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
AddFileResult result = await dropClient.AddFileAsync(dropItem);
|
||||
return IpcResult.Success(Inv(
|
||||
"File '{0}' {1} under '{2}' in drop '{3}'.",
|
||||
dropItem.FullFilePath,
|
||||
result,
|
||||
dropItem.RelativeDropPath,
|
||||
DropName));
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets file's path relative to a given root.
|
||||
/// The method assumes that file is under the root; however it does not enforce this assumption.
|
||||
/// </summary>
|
||||
private string GetRelativePath(string root, string file)
|
||||
{
|
||||
var rootEndsWithSlash =
|
||||
root[root.Length - 1] == Path.DirectorySeparatorChar
|
||||
|| root[root.Length - 1] == Path.AltDirectorySeparatorChar;
|
||||
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous version of <see cref="FinalizeAsync"/>
|
||||
/// </summary>
|
||||
public IIpcResult Finalize()
|
||||
{
|
||||
return FinalizeAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
|
||||
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
|
||||
/// result of this operation.
|
||||
/// </summary>
|
||||
public async Task<IIpcResult> FinalizeAsync()
|
||||
{
|
||||
var dropFinalizationEvent =
|
||||
await SendDropEtwEvent(
|
||||
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
|
||||
|
||||
return dropFinalizationEvent.Succeeded
|
||||
? IpcResult.Success(Inv("Drop {0} finalized", DropName))
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, dropFinalizationEvent.ErrorMessage);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
|
||||
{
|
||||
ReportStatisticsAsync().GetAwaiter().GetResult();
|
||||
|
||||
m_dropClientTask.Result.Dispose();
|
||||
}
|
||||
|
||||
m_server.Dispose();
|
||||
ApiClient?.Dispose();
|
||||
m_logger.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop create' operation by delegating to <see cref="IDropClient.CreateAsync"/>.
|
||||
///
|
||||
/// If successful, returns <see cref="DropCreationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
|
||||
/// set to true, <see cref="DropCreationEvent.DropExpirationInDays"/> set to drop expiration in days,
|
||||
/// and <see cref="DropOperationBaseEvent.AdditionalInformation"/> set to the textual representation
|
||||
/// of the returned <see cref="DropItem"/> object.
|
||||
///
|
||||
/// Doesn't handle any exceptions.
|
||||
/// </summary>
|
||||
private async Task<DropCreationEvent> InternalCreateAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
DropItem dropItem = await dropClient.CreateAsync();
|
||||
return new DropCreationEvent()
|
||||
{
|
||||
Succeeded = true,
|
||||
AdditionalInformation = DropItemToString(dropItem),
|
||||
DropExpirationInDays = ComputeDropItemExpiration(dropItem),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop finalize' operation by delegating to <see cref="IDropClient.FinalizeAsync"/>.
|
||||
///
|
||||
/// If successful, returns <see cref="DropFinalizationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
|
||||
/// set to true.
|
||||
///
|
||||
/// Doesn't handle any exceptions.
|
||||
/// </summary>
|
||||
private async Task<DropFinalizationEvent> InternalFinalizeAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
await dropClient.FinalizeAsync();
|
||||
return new DropFinalizationEvent()
|
||||
{
|
||||
Succeeded = true,
|
||||
};
|
||||
}
|
||||
|
||||
private async Task ReportStatisticsAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
var stats = dropClient.GetStats();
|
||||
if (stats != null && stats.Any())
|
||||
{
|
||||
// log stats
|
||||
m_logger.Info("Statistics: ");
|
||||
m_logger.Info(string.Join(Environment.NewLine, stats.Select(s => s.Key + " = " + s.Value)));
|
||||
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.MaxParallelUploads), DropConfig.MaxParallelUploads);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.NagleTime), (long)DropConfig.NagleTime.TotalMilliseconds);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.BatchSize), DropConfig.BatchSize);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.EnableChunkDedup), DropConfig.EnableChunkDedup ? 1 : 0);
|
||||
stats.Add("DaemonConfig" + nameof(Config.MaxConcurrentClients), Config.MaxConcurrentClients);
|
||||
stats.Add("DaemonConfig" + nameof(Config.ConnectRetryDelay), (long)Config.ConnectRetryDelay.TotalMilliseconds);
|
||||
stats.Add("DaemonConfig" + nameof(Config.MaxConnectRetries), Config.MaxConnectRetries);
|
||||
|
||||
stats.AddRange(m_counters.AsStatistics());
|
||||
|
||||
// report stats to BuildXL (if m_client is specified)
|
||||
if (ApiClient != null)
|
||||
{
|
||||
var possiblyReported = await ApiClient.ReportStatistics(stats);
|
||||
if (possiblyReported.Succeeded && possiblyReported.Result)
|
||||
{
|
||||
m_logger.Info("Statistics successfully reported to BuildXL.");
|
||||
}
|
||||
else
|
||||
{
|
||||
var errorDescription = possiblyReported.Succeeded ? string.Empty : possiblyReported.Failure.Describe();
|
||||
m_logger.Warning("Reporting stats to BuildXL failed. " + errorDescription);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_logger.Info("No stats recorded by drop client of type " + dropClient.GetType().Name);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task<IIpcResult> WrapDropErrorsIntoIpcResult(Func<Task<IIpcResult>> factory)
|
||||
{
|
||||
return HandleKnownErrors(
|
||||
factory,
|
||||
(errorMessage) => new IpcResult(IpcResultStatus.ExecutionError, errorMessage));
|
||||
}
|
||||
|
||||
private static Task<TDropEvent> WrapDropErrorsIntoDropEtwEvent<TDropEvent>(Func<Task<TDropEvent>> factory) where TDropEvent : DropOperationBaseEvent
|
||||
{
|
||||
return HandleKnownErrors(
|
||||
factory,
|
||||
(errorMessage) =>
|
||||
{
|
||||
var dropEvent = Activator.CreateInstance<TDropEvent>();
|
||||
dropEvent.Succeeded = false;
|
||||
dropEvent.ErrorMessage = errorMessage;
|
||||
return dropEvent;
|
||||
});
|
||||
}
|
||||
|
||||
private static async Task<TResult> HandleKnownErrors<TResult>(Func<Task<TResult>> factory, Func<string, TResult> errorValueFactory)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await factory();
|
||||
}
|
||||
catch (VssUnauthorizedException e)
|
||||
{
|
||||
return errorValueFactory("[DROP AUTH ERROR] " + e.Message);
|
||||
}
|
||||
catch (DropServiceException e)
|
||||
{
|
||||
return errorValueFactory("[DROP SERVICE ERROR] " + e.Message);
|
||||
}
|
||||
catch (DropDaemonException e)
|
||||
{
|
||||
return errorValueFactory("[DROP DAEMON ERROR] " + e.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static string DropItemToString(DropItem dropItem)
|
||||
{
|
||||
try
|
||||
{
|
||||
return dropItem?.ToJson().ToString();
|
||||
}
|
||||
#pragma warning disable ERP022 // TODO: This should really handle specific errors
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
|
||||
}
|
||||
|
||||
private static int ComputeDropItemExpiration(DropItem dropItem)
|
||||
{
|
||||
DateTime? expirationDate;
|
||||
return dropItem.TryGetExpirationTime(out expirationDate) || expirationDate.HasValue
|
||||
? (int)expirationDate.Value.Subtract(DateTime.UtcNow).TotalDays
|
||||
: -1;
|
||||
}
|
||||
|
||||
private async Task<T> SendDropEtwEvent<T>(Task<T> task) where T : DropOperationBaseEvent
|
||||
{
|
||||
long startTime = DateTime.UtcNow.Ticks;
|
||||
T dropEvent = null;
|
||||
try
|
||||
{
|
||||
dropEvent = await task;
|
||||
return dropEvent;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// if 'task' failed, create an event indicating an error
|
||||
if (dropEvent == null)
|
||||
{
|
||||
dropEvent = Activator.CreateInstance<T>();
|
||||
dropEvent.Succeeded = false;
|
||||
dropEvent.ErrorMessage = "internal error";
|
||||
}
|
||||
|
||||
// common properties: execution time, drop type, drop url
|
||||
dropEvent.ElapsedTimeTicks = DateTime.UtcNow.Ticks - startTime;
|
||||
dropEvent.DropType = "VsoDrop";
|
||||
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
|
||||
{
|
||||
dropEvent.DropUrl = (await m_dropClientTask).DropUrl;
|
||||
}
|
||||
|
||||
// send event
|
||||
m_etwLogger.Log(dropEvent);
|
||||
}
|
||||
}
|
||||
|
||||
private readonly CounterCollection<DaemonCounter> m_counters = new CounterCollection<DaemonCounter>();
|
||||
|
||||
private enum DaemonCounter
|
||||
{
|
||||
/// <nodoc/>
|
||||
[CounterType(CounterType.Stopwatch)]
|
||||
ParseArgsDuration,
|
||||
|
||||
/// <nodoc/>
|
||||
[CounterType(CounterType.Stopwatch)]
|
||||
ServerActionDuration,
|
||||
|
||||
/// <nodoc/>
|
||||
QueueDurationMs,
|
||||
}
|
||||
|
||||
private async Task<IIpcResult> ParseAndExecuteCommand(IIpcOperation operation)
|
||||
{
|
||||
string cmdLine = operation.Payload;
|
||||
m_logger.Verbose("Command received: {0}", cmdLine);
|
||||
ConfiguredCommand conf;
|
||||
using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration))
|
||||
{
|
||||
conf = Program.ParseArgs(cmdLine, m_parser);
|
||||
}
|
||||
|
||||
IIpcResult result;
|
||||
using (m_counters.StartStopwatch(DaemonCounter.ServerActionDuration))
|
||||
{
|
||||
result = await conf.Command.ServerAction(conf, this);
|
||||
}
|
||||
|
||||
TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime;
|
||||
m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
Task<IIpcResult> IIpcOperationExecutor.ExecuteAsync(IIpcOperation operation)
|
||||
{
|
||||
Contract.Requires(operation != null);
|
||||
|
||||
return ParseAndExecuteCommand(operation);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,914 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Reflection;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.ExternalApi;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Scheduler;
|
||||
using BuildXL.Storage;
|
||||
using BuildXL.Tracing.CloudBuild;
|
||||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.CLI;
|
||||
using BuildXL.Utilities.Tasks;
|
||||
using Microsoft.VisualStudio.Services.Common;
|
||||
using Microsoft.VisualStudio.Services.Drop.WebApi;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Tool.ServicePipDaemon;
|
||||
using static BuildXL.Utilities.FormattableStringEx;
|
||||
using static Tool.ServicePipDaemon.Statics;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Responsible for accepting and handling TCP/IP connections from clients.
|
||||
/// </summary>
|
||||
public sealed class DropDaemon : ServicePipDaemon.ServicePipDaemon, IDisposable, IIpcOperationExecutor
|
||||
{
|
||||
private const int ServicePointParallelismForDrop = 200;
|
||||
|
||||
private const string IncludeAllFilter = ".*";
|
||||
|
||||
/// <summary>Prefix for the error message of the exception that gets thrown when a symlink is attempted to be added to drop.</summary>
|
||||
internal const string SymlinkAddErrorMessagePrefix = "SymLinks may not be added to drop: ";
|
||||
|
||||
private const string LogFileName = "DropDaemon";
|
||||
|
||||
/// <nodoc/>
|
||||
public const string DropDLogPrefix = "(DropD) ";
|
||||
|
||||
private static readonly int s_minIoThreadsForDrop = Environment.ProcessorCount * 10;
|
||||
|
||||
private static readonly int s_minWorkerThreadsForDrop = Environment.ProcessorCount * 10;
|
||||
|
||||
internal static readonly List<Option> DropConfigOptions = new List<Option>();
|
||||
|
||||
/// <summary>Drop configuration.</summary>
|
||||
public DropConfig DropConfig { get; }
|
||||
|
||||
/// <summary>Name of the drop this daemon is constructing.</summary>
|
||||
public string DropName => DropConfig.Name;
|
||||
|
||||
private readonly Task<IDropClient> m_dropClientTask;
|
||||
|
||||
internal static IEnumerable<Command> SupportedCommands => Commands.Values;
|
||||
|
||||
#region Options and commands
|
||||
|
||||
internal static readonly StrOption DropServiceConfigFile = RegisterDaemonConfigOption(new StrOption("dropServiceConfigFile")
|
||||
{
|
||||
ShortName = "c",
|
||||
HelpText = "Drop service configuration file",
|
||||
DefaultValue = null,
|
||||
Expander = (fileName) =>
|
||||
{
|
||||
var json = System.IO.File.ReadAllText(fileName);
|
||||
var jObject = JObject.Parse(json);
|
||||
return jObject.Properties().Select(prop => new ParsedOption(PrefixKind.Long, prop.Name, prop.Value.ToString()));
|
||||
},
|
||||
});
|
||||
|
||||
internal static readonly StrOption DropNameOption = RegisterDropConfigOption(new StrOption("name")
|
||||
{
|
||||
ShortName = "n",
|
||||
HelpText = "Drop name",
|
||||
IsRequired = true,
|
||||
});
|
||||
|
||||
internal static readonly UriOption DropEndpoint = RegisterDropConfigOption(new UriOption("service")
|
||||
{
|
||||
ShortName = "s",
|
||||
HelpText = "Drop endpoint URI",
|
||||
IsRequired = true,
|
||||
});
|
||||
|
||||
internal static readonly IntOption BatchSize = RegisterDropConfigOption(new IntOption("batchSize")
|
||||
{
|
||||
ShortName = "bs",
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Size of batches in which to send 'associate' requests)",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultBatchSizeForAssociate,
|
||||
});
|
||||
|
||||
internal static readonly IntOption MaxParallelUploads = RegisterDropConfigOption(new IntOption("maxParallelUploads")
|
||||
{
|
||||
ShortName = "mpu",
|
||||
HelpText = "Maximum number of uploads to issue to drop service in parallel",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultMaxParallelUploads,
|
||||
});
|
||||
|
||||
internal static readonly IntOption NagleTimeMillis = RegisterDropConfigOption(new IntOption("nagleTimeMillis")
|
||||
{
|
||||
ShortName = "nt",
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Maximum time in milliseconds to wait before triggering a batch 'associate' request)",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultNagleTimeForAssociate.TotalMilliseconds,
|
||||
});
|
||||
|
||||
internal static readonly IntOption RetentionDays = RegisterDropConfigOption(new IntOption("retentionDays")
|
||||
{
|
||||
ShortName = "rt",
|
||||
HelpText = "Drop retention time in days",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultRetention.TotalDays,
|
||||
});
|
||||
|
||||
internal static readonly IntOption HttpSendTimeoutMillis = RegisterDropConfigOption(new IntOption("httpSendTimeoutMillis")
|
||||
{
|
||||
HelpText = "Timeout for http requests",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultHttpSendTimeout.TotalMilliseconds,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption EnableTelemetry = RegisterDropConfigOption(new BoolOption("enableTelemetry")
|
||||
{
|
||||
ShortName = "t",
|
||||
HelpText = "Verbose logging",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultEnableTelemetry,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption EnableChunkDedup = RegisterDropConfigOption(new BoolOption("enableChunkDedup")
|
||||
{
|
||||
ShortName = "cd",
|
||||
HelpText = "Chunk level dedup",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultEnableChunkDedup,
|
||||
});
|
||||
|
||||
// ==============================================================================
|
||||
// 'addfile' and 'addartifacts' parameters
|
||||
// ==============================================================================
|
||||
internal static readonly StrOption RelativeDropPath = new StrOption("dropPath")
|
||||
{
|
||||
ShortName = "d",
|
||||
HelpText = "Relative drop path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption HashOptional = new StrOption("hash")
|
||||
{
|
||||
ShortName = "h",
|
||||
HelpText = "VSO file hash",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption FileId = new StrOption("fileId")
|
||||
{
|
||||
ShortName = "fid",
|
||||
HelpText = "BuildXL file identifier",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption Directory = new StrOption("directory")
|
||||
{
|
||||
ShortName = "dir",
|
||||
HelpText = "Directory path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption DirectoryId = new StrOption("directoryId")
|
||||
{
|
||||
ShortName = "dirid",
|
||||
HelpText = "BuildXL directory identifier",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption RelativeDirectoryDropPath = new StrOption("directoryDropPath")
|
||||
{
|
||||
ShortName = "dird",
|
||||
HelpText = "Relative drop path for directory",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption DirectoryContentFilter = new StrOption("directoryFilter")
|
||||
{
|
||||
ShortName = "dcfilter",
|
||||
HelpText = "Directory content filter (only files that match the filter will be added to drop).",
|
||||
DefaultValue = null,
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly Command StartNoDropCmd = RegisterCommand(
|
||||
name: "start-nodrop",
|
||||
description: @"Starts a server process without a backing VSO drop client (useful for testing/pinging the daemon).",
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
var dropConfig = new DropConfig(string.Empty, new Uri("file://xyz"));
|
||||
var daemonConfig = CreateDaemonConfig(conf);
|
||||
var vsoClientTask = TaskSourceSlim.Create<IDropClient>();
|
||||
vsoClientTask.SetException(new NotSupportedException());
|
||||
using (var daemon = new DropDaemon(conf.Config.Parser, daemonConfig, dropConfig, vsoClientTask.Task))
|
||||
{
|
||||
daemon.Start();
|
||||
daemon.Completion.GetAwaiter().GetResult();
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
internal static readonly Command StartCmd = RegisterCommand(
|
||||
name: "start",
|
||||
description: "Starts the server process.",
|
||||
options: DropConfigOptions.Union(new[] { IpcServerMonikerOptional }),
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
SetupThreadPoolAndServicePoint();
|
||||
var dropConfig = CreateDropConfig(conf);
|
||||
var daemonConf = CreateDaemonConfig(conf);
|
||||
|
||||
if (daemonConf.MaxConcurrentClients <= 1)
|
||||
{
|
||||
conf.Logger.Error($"Must specify at least 2 '{nameof(DaemonConfig.MaxConcurrentClients)}' when running DropDaemon to avoid deadlock when stopping this daemon from a different client");
|
||||
return -1;
|
||||
}
|
||||
|
||||
using (var client = CreateClient(conf.Get(IpcServerMonikerOptional), daemonConf))
|
||||
using (var daemon = new DropDaemon(
|
||||
parser: conf.Config.Parser,
|
||||
daemonConfig: daemonConf,
|
||||
dropConfig: dropConfig,
|
||||
dropClientTask: null,
|
||||
client: client))
|
||||
{
|
||||
daemon.Start();
|
||||
daemon.Completion.GetAwaiter().GetResult();
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
private static void SetupThreadPoolAndServicePoint()
|
||||
{
|
||||
int workerThreads, ioThreads;
|
||||
ThreadPool.GetMinThreads(out workerThreads, out ioThreads);
|
||||
|
||||
workerThreads = Math.Max(workerThreads, s_minWorkerThreadsForDrop);
|
||||
ioThreads = Math.Max(ioThreads, s_minIoThreadsForDrop);
|
||||
ThreadPool.SetMinThreads(workerThreads, ioThreads);
|
||||
|
||||
ServicePointManager.DefaultConnectionLimit = Math.Max(ServicePointParallelismForDrop, ServicePointManager.DefaultConnectionLimit);
|
||||
}
|
||||
|
||||
internal static readonly Command StartDaemonCmd = RegisterCommand(
|
||||
name: "start-daemon",
|
||||
description: "Starts the server process in background (as daemon).",
|
||||
options: DropConfigOptions,
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
using (var daemon = new Process())
|
||||
{
|
||||
bool shellExecute = conf.Get(ShellExecute);
|
||||
daemon.StartInfo.FileName = AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetEntryAssembly());
|
||||
daemon.StartInfo.Arguments = "start " + conf.Config.Render();
|
||||
daemon.StartInfo.LoadUserProfile = false;
|
||||
daemon.StartInfo.UseShellExecute = shellExecute;
|
||||
daemon.StartInfo.CreateNoWindow = !shellExecute;
|
||||
daemon.Start();
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
internal static readonly Command CreateDropCmd = RegisterCommand(
|
||||
name: "create",
|
||||
description: "[RPC] Invokes the 'create' operation.",
|
||||
options: DropConfigOptions,
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, dropDaemon) =>
|
||||
{
|
||||
var daemon = dropDaemon as DropDaemon;
|
||||
daemon.Logger.Info("[CREATE]: Started at " + daemon.DropConfig.Service + "/" + daemon.DropName);
|
||||
IIpcResult result = await daemon.CreateAsync();
|
||||
daemon.Logger.Info("[CREATE]: " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command FinalizeDropCmd = RegisterCommand(
|
||||
name: "finalize",
|
||||
description: "[RPC] Invokes the 'finalize' operation.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, dropDaemon) =>
|
||||
{
|
||||
var daemon = dropDaemon as DropDaemon;
|
||||
daemon.Logger.Info("[FINALIZE] Started at" + daemon.DropConfig.Service + "/" + daemon.DropName);
|
||||
IIpcResult result = await daemon.FinalizeAsync();
|
||||
daemon.Logger.Info("[FINALIZE] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command FinalizeDropAndStopDaemonCmd = RegisterCommand(
|
||||
name: "finalize-and-stop",
|
||||
description: "[RPC] Invokes the 'finalize' operation; then stops the daemon.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: Command.Compose(FinalizeDropCmd.ServerAction, StopDaemonCmd.ServerAction));
|
||||
|
||||
internal static readonly Command AddFileToDropCmd = RegisterCommand(
|
||||
name: "addfile",
|
||||
description: "[RPC] invokes the 'addfile' operation.",
|
||||
options: new Option[] { File, RelativeDropPath, HashOptional },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, dropDaemon) =>
|
||||
{
|
||||
var daemon = dropDaemon as DropDaemon;
|
||||
daemon.Logger.Verbose("[ADDFILE] Started");
|
||||
string filePath = conf.Get(File);
|
||||
string hashValue = conf.Get(HashOptional);
|
||||
var contentInfo = string.IsNullOrEmpty(hashValue) ? null : (FileContentInfo?)FileContentInfo.Parse(hashValue);
|
||||
var dropItem = new DropItemForFile(filePath, conf.Get(RelativeDropPath), contentInfo);
|
||||
IIpcResult result = System.IO.File.Exists(filePath)
|
||||
? await daemon.AddFileAsync(dropItem)
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, "file '" + filePath + "' does not exist");
|
||||
daemon.Logger.Verbose("[ADDFILE] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command AddArtifactsToDropCmd = RegisterCommand(
|
||||
name: "addartifacts",
|
||||
description: "[RPC] invokes the 'addartifacts' operation.",
|
||||
options: new Option[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath, Directory, DirectoryId, RelativeDirectoryDropPath, DirectoryContentFilter },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, dropDaemon) =>
|
||||
{
|
||||
var daemon = dropDaemon as DropDaemon;
|
||||
daemon.Logger.Verbose("[ADDARTIFACTS] Started");
|
||||
|
||||
var result = await AddArtifactsToDropInternalAsync(conf, daemon);
|
||||
|
||||
daemon.Logger.Verbose("[ADDARTIFACTS] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// The purpose of this ctor is to force 'predictable' initialization of static fields.
|
||||
/// </summary>
|
||||
static DropDaemon()
|
||||
{
|
||||
// noop
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
public DropDaemon(IParser parser, DaemonConfig daemonConfig, DropConfig dropConfig, Task<IDropClient> dropClientTask, IIpcProvider rpcProvider = null, Client client = null)
|
||||
: base(parser,
|
||||
daemonConfig,
|
||||
!string.IsNullOrWhiteSpace(dropConfig.LogDir) ? new FileLogger(dropConfig.LogDir, LogFileName, daemonConfig.Moniker, dropConfig.Verbose, DropDLogPrefix) : daemonConfig.Logger,
|
||||
rpcProvider,
|
||||
client)
|
||||
{
|
||||
Contract.Requires(dropConfig != null);
|
||||
|
||||
DropConfig = dropConfig;
|
||||
m_logger.Info("Using DropDaemon config: " + JsonConvert.SerializeObject(Config));
|
||||
|
||||
m_dropClientTask = dropClientTask ?? Task.Run(() => (IDropClient)new VsoClient(m_logger, dropConfig));
|
||||
}
|
||||
|
||||
internal static void EnsureCommandsInitialized()
|
||||
{
|
||||
Contract.Assert(Commands != null);
|
||||
|
||||
// these operations are quite expensive, however, we expect to call this method only once per drop, so it should cause any perf downgrade
|
||||
var numCommandsBase = typeof(ServicePipDaemon.ServicePipDaemon).GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(f => f.FieldType == typeof(Command)).Count();
|
||||
var numCommandsDropD = typeof(DropDaemon).GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static).Where(f => f.FieldType == typeof(Command)).Count();
|
||||
|
||||
if (Commands.Count != numCommandsBase + numCommandsDropD)
|
||||
{
|
||||
Contract.Assert(false, $"The list of commands was not properly initialized (# of initialized commands = {Commands.Count}; # of ServicePipDaemon commands = {numCommandsBase}; # of DropDaemon commands = {numCommandsDropD})");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous version of <see cref="CreateAsync"/>
|
||||
/// </summary>
|
||||
public IIpcResult Create()
|
||||
{
|
||||
return CreateAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the drop. Handles drop-related exceptions by omitting their stack traces.
|
||||
/// In all cases emits an appropriate <see cref="DropCreationEvent"/> indicating the
|
||||
/// result of this operation.
|
||||
/// </summary>
|
||||
public async Task<IIpcResult> CreateAsync()
|
||||
{
|
||||
DropCreationEvent dropCreationEvent =
|
||||
await SendDropEtwEvent(
|
||||
WrapDropErrorsIntoDropEtwEvent(InternalCreateAsync));
|
||||
|
||||
return dropCreationEvent.Succeeded
|
||||
? IpcResult.Success(I($"Drop {DropName} created."))
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, dropCreationEvent.ErrorMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop addfile' operation by delegating to <see cref="IDropClient.AddFileAsync"/>.
|
||||
/// Handles drop-related exceptions by omitting their stack traces.
|
||||
/// </summary>
|
||||
public Task<IIpcResult> AddFileAsync(IDropItem dropItem)
|
||||
{
|
||||
return AddFileAsync(dropItem, IsSymLinkOrMountPoint);
|
||||
}
|
||||
|
||||
internal async Task<IIpcResult> AddFileAsync(IDropItem dropItem, Func<string, bool> symlinkTester)
|
||||
{
|
||||
Contract.Requires(dropItem != null);
|
||||
|
||||
// Check if the file is a symlink, only if the file exists on disk at this point; if it is a symlink, reject it outright.
|
||||
if (System.IO.File.Exists(dropItem.FullFilePath) && symlinkTester(dropItem.FullFilePath))
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, SymlinkAddErrorMessagePrefix + dropItem.FullFilePath);
|
||||
}
|
||||
|
||||
return await WrapDropErrorsIntoIpcResult(async () =>
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
AddFileResult result = await dropClient.AddFileAsync(dropItem);
|
||||
return IpcResult.Success(I($"File '{dropItem.FullFilePath}' {result} under '{dropItem.RelativeDropPath}' in drop '{DropName}'."));
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Synchronous version of <see cref="FinalizeAsync"/>
|
||||
/// </summary>
|
||||
public IIpcResult Finalize()
|
||||
{
|
||||
return FinalizeAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finalizes the drop. Handles drop-related exceptions by omitting their stack traces.
|
||||
/// In all cases emits an appropriate <see cref="DropFinalizationEvent"/> indicating the
|
||||
/// result of this operation.
|
||||
/// </summary>
|
||||
public async Task<IIpcResult> FinalizeAsync()
|
||||
{
|
||||
var dropFinalizationEvent =
|
||||
await SendDropEtwEvent(
|
||||
WrapDropErrorsIntoDropEtwEvent(InternalFinalizeAsync));
|
||||
|
||||
return dropFinalizationEvent.Succeeded
|
||||
? IpcResult.Success(I($"Drop {DropName} finalized"))
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, dropFinalizationEvent.ErrorMessage);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public new void Dispose()
|
||||
{
|
||||
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
|
||||
{
|
||||
ReportStatisticsAsync().GetAwaiter().GetResult();
|
||||
|
||||
m_dropClientTask.Result.Dispose();
|
||||
}
|
||||
|
||||
base.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop create' operation by delegating to <see cref="IDropClient.CreateAsync"/>.
|
||||
///
|
||||
/// If successful, returns <see cref="DropCreationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
|
||||
/// set to true, <see cref="DropCreationEvent.DropExpirationInDays"/> set to drop expiration in days,
|
||||
/// and <see cref="DropOperationBaseEvent.AdditionalInformation"/> set to the textual representation
|
||||
/// of the returned <see cref="DropItem"/> object.
|
||||
///
|
||||
/// Doesn't handle any exceptions.
|
||||
/// </summary>
|
||||
private async Task<DropCreationEvent> InternalCreateAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
DropItem dropItem = await dropClient.CreateAsync();
|
||||
return new DropCreationEvent()
|
||||
{
|
||||
Succeeded = true,
|
||||
AdditionalInformation = DropItemToString(dropItem),
|
||||
DropExpirationInDays = ComputeDropItemExpiration(dropItem),
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Invokes the 'drop finalize' operation by delegating to <see cref="IDropClient.FinalizeAsync"/>.
|
||||
///
|
||||
/// If successful, returns <see cref="DropFinalizationEvent"/> with <see cref="DropOperationBaseEvent.Succeeded"/>
|
||||
/// set to true.
|
||||
///
|
||||
/// Doesn't handle any exceptions.
|
||||
/// </summary>
|
||||
private async Task<DropFinalizationEvent> InternalFinalizeAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
await dropClient.FinalizeAsync();
|
||||
return new DropFinalizationEvent()
|
||||
{
|
||||
Succeeded = true,
|
||||
};
|
||||
}
|
||||
|
||||
private async Task ReportStatisticsAsync()
|
||||
{
|
||||
IDropClient dropClient = await m_dropClientTask;
|
||||
var stats = dropClient.GetStats();
|
||||
if (stats != null && stats.Any())
|
||||
{
|
||||
// log stats
|
||||
m_logger.Info("Statistics: ");
|
||||
m_logger.Info(string.Join(Environment.NewLine, stats.Select(s => s.Key + " = " + s.Value)));
|
||||
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.MaxParallelUploads), DropConfig.MaxParallelUploads);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.NagleTime), (long)DropConfig.NagleTime.TotalMilliseconds);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.BatchSize), DropConfig.BatchSize);
|
||||
stats.Add(nameof(DropConfig) + nameof(DropConfig.EnableChunkDedup), DropConfig.EnableChunkDedup ? 1 : 0);
|
||||
stats.Add(nameof(DaemonConfig) + nameof(Config.MaxConcurrentClients), Config.MaxConcurrentClients);
|
||||
stats.Add(nameof(DaemonConfig) + nameof(Config.ConnectRetryDelay), (long)Config.ConnectRetryDelay.TotalMilliseconds);
|
||||
stats.Add(nameof(DaemonConfig) + nameof(Config.MaxConnectRetries), Config.MaxConnectRetries);
|
||||
|
||||
stats.AddRange(m_counters.AsStatistics());
|
||||
|
||||
// report stats to BuildXL (if m_client is specified)
|
||||
if (ApiClient != null)
|
||||
{
|
||||
var possiblyReported = await ApiClient.ReportStatistics(stats);
|
||||
if (possiblyReported.Succeeded && possiblyReported.Result)
|
||||
{
|
||||
m_logger.Info("Statistics successfully reported to BuildXL.");
|
||||
}
|
||||
else
|
||||
{
|
||||
var errorDescription = possiblyReported.Succeeded ? string.Empty : possiblyReported.Failure.Describe();
|
||||
m_logger.Warning("Reporting stats to BuildXL failed. " + errorDescription);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
m_logger.Info("No stats recorded by drop client of type " + dropClient.GetType().Name);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task<IIpcResult> WrapDropErrorsIntoIpcResult(Func<Task<IIpcResult>> factory)
|
||||
{
|
||||
return HandleKnownErrors(
|
||||
factory,
|
||||
(errorMessage) => new IpcResult(IpcResultStatus.ExecutionError, errorMessage));
|
||||
}
|
||||
|
||||
private static Task<TDropEvent> WrapDropErrorsIntoDropEtwEvent<TDropEvent>(Func<Task<TDropEvent>> factory) where TDropEvent : DropOperationBaseEvent
|
||||
{
|
||||
return HandleKnownErrors(
|
||||
factory,
|
||||
(errorMessage) =>
|
||||
{
|
||||
var dropEvent = Activator.CreateInstance<TDropEvent>();
|
||||
dropEvent.Succeeded = false;
|
||||
dropEvent.ErrorMessage = errorMessage;
|
||||
return dropEvent;
|
||||
});
|
||||
}
|
||||
|
||||
private static async Task<TResult> HandleKnownErrors<TResult>(Func<Task<TResult>> factory, Func<string, TResult> errorValueFactory)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await factory();
|
||||
}
|
||||
catch (VssUnauthorizedException e)
|
||||
{
|
||||
return errorValueFactory("[DROP AUTH ERROR] " + e.Message);
|
||||
}
|
||||
catch (DropServiceException e)
|
||||
{
|
||||
return errorValueFactory("[DROP SERVICE ERROR] " + e.Message);
|
||||
}
|
||||
catch (DaemonException e)
|
||||
{
|
||||
return errorValueFactory("[DAEMON ERROR] " + e.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static string DropItemToString(DropItem dropItem)
|
||||
{
|
||||
try
|
||||
{
|
||||
return dropItem?.ToJson().ToString();
|
||||
}
|
||||
#pragma warning disable ERP022 // TODO: This should really handle specific errors
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
|
||||
}
|
||||
|
||||
private static int ComputeDropItemExpiration(DropItem dropItem)
|
||||
{
|
||||
DateTime? expirationDate;
|
||||
return dropItem.TryGetExpirationTime(out expirationDate) || expirationDate.HasValue
|
||||
? (int)expirationDate.Value.Subtract(DateTime.UtcNow).TotalDays
|
||||
: -1;
|
||||
}
|
||||
|
||||
private async Task<T> SendDropEtwEvent<T>(Task<T> task) where T : DropOperationBaseEvent
|
||||
{
|
||||
long startTime = DateTime.UtcNow.Ticks;
|
||||
T dropEvent = null;
|
||||
try
|
||||
{
|
||||
dropEvent = await task;
|
||||
return dropEvent;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// if 'task' failed, create an event indicating an error
|
||||
if (dropEvent == null)
|
||||
{
|
||||
dropEvent = Activator.CreateInstance<T>();
|
||||
dropEvent.Succeeded = false;
|
||||
dropEvent.ErrorMessage = "internal error";
|
||||
}
|
||||
|
||||
// common properties: execution time, drop type, drop url
|
||||
dropEvent.ElapsedTimeTicks = DateTime.UtcNow.Ticks - startTime;
|
||||
dropEvent.DropType = "VsoDrop";
|
||||
if (m_dropClientTask.IsCompleted && !m_dropClientTask.IsFaulted)
|
||||
{
|
||||
dropEvent.DropUrl = (await m_dropClientTask).DropUrl;
|
||||
}
|
||||
|
||||
// send event
|
||||
m_etwLogger.Log(dropEvent);
|
||||
}
|
||||
}
|
||||
|
||||
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
|
||||
{
|
||||
return new DropConfig(
|
||||
dropName: conf.Get(DropNameOption),
|
||||
serviceEndpoint: conf.Get(DropEndpoint),
|
||||
maxParallelUploads: conf.Get(MaxParallelUploads),
|
||||
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
|
||||
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
|
||||
verbose: conf.Get(Verbose),
|
||||
enableTelemetry: conf.Get(EnableTelemetry),
|
||||
enableChunkDedup: conf.Get(EnableChunkDedup),
|
||||
logDir: conf.Get(LogDir));
|
||||
}
|
||||
|
||||
private static T RegisterDropConfigOption<T>(T option) where T : Option => RegisterOption(DropConfigOptions, option);
|
||||
|
||||
private static Client CreateClient(string serverMoniker, IClientConfig config)
|
||||
{
|
||||
return serverMoniker != null
|
||||
? Client.Create(serverMoniker, config)
|
||||
: null;
|
||||
}
|
||||
|
||||
private static async Task<IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, DropDaemon daemon)
|
||||
{
|
||||
var files = File.GetValues(conf.Config).ToArray();
|
||||
var fileIds = FileId.GetValues(conf.Config).ToArray();
|
||||
var hashes = HashOptional.GetValues(conf.Config).ToArray();
|
||||
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
|
||||
|
||||
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.GenericError,
|
||||
I($"File counts don't match: #files = {files.Length}, #fileIds = {fileIds.Length}, #hashes = {hashes.Length}, #dropPaths = {dropPaths.Length}"));
|
||||
}
|
||||
|
||||
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
|
||||
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
|
||||
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
|
||||
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
|
||||
|
||||
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.GenericError,
|
||||
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #dropPaths = {directoryDropPaths.Length}, #directoryFilters = {directoryFilters.Length}"));
|
||||
}
|
||||
|
||||
(Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
|
||||
if (filterInitError != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, filterInitError);
|
||||
}
|
||||
|
||||
var dropFileItemsKeyedByIsAbsent = Enumerable
|
||||
.Range(0, files.Length)
|
||||
.Select(i => new DropItemForBuildXLFile(
|
||||
daemon.ApiClient,
|
||||
chunkDedup: conf.Get(EnableChunkDedup),
|
||||
filePath: files[i],
|
||||
fileId: fileIds[i],
|
||||
fileContentInfo: FileContentInfo.Parse(hashes[i]),
|
||||
relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
|
||||
|
||||
// If a user specified a particular file to be added to drop, this file must be a part of drop.
|
||||
// The missing files will not get into the drop, so we emit an error.
|
||||
if (dropFileItemsKeyedByIsAbsent[true].Any())
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.InvalidInput,
|
||||
I($"The following files are missing, but they are a part of the drop command:{Environment.NewLine}{string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])}"));
|
||||
}
|
||||
|
||||
(IEnumerable<DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
|
||||
conf,
|
||||
daemon,
|
||||
directoryPaths,
|
||||
directoryIds,
|
||||
directoryDropPaths,
|
||||
initializedFilters);
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, error);
|
||||
}
|
||||
|
||||
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
|
||||
|
||||
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
|
||||
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.InvalidInput,
|
||||
I($"Uploading missing source file(s) is not supported:{Environment.NewLine}{string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))}"));
|
||||
}
|
||||
|
||||
// return early if there is nothing to upload
|
||||
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.Success, string.Empty);
|
||||
}
|
||||
|
||||
return await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]));
|
||||
}
|
||||
|
||||
private static (Regex[], string error) InitializeDirectoryFilters(string[] filters)
|
||||
{
|
||||
try
|
||||
{
|
||||
var initializedFilters = filters.Select(
|
||||
filter => filter == IncludeAllFilter
|
||||
? null
|
||||
: new Regex(filter, RegexOptions.Compiled | RegexOptions.IgnoreCase));
|
||||
|
||||
return (initializedFilters.ToArray(), null);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return (null, e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
|
||||
ConfiguredCommand conf,
|
||||
DropDaemon daemon,
|
||||
string directoryPath,
|
||||
string directoryId,
|
||||
string dropPath,
|
||||
Regex contentFilter)
|
||||
{
|
||||
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
|
||||
Contract.Requires(!string.IsNullOrEmpty(directoryId));
|
||||
Contract.Requires(dropPath != null);
|
||||
|
||||
if (daemon.ApiClient == null)
|
||||
{
|
||||
return (null, "ApiClient is not initialized");
|
||||
}
|
||||
|
||||
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
|
||||
|
||||
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
|
||||
if (!maybeResult.Succeeded)
|
||||
{
|
||||
return (null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
|
||||
}
|
||||
|
||||
var directoryContent = maybeResult.Result;
|
||||
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
|
||||
|
||||
if (contentFilter != null)
|
||||
{
|
||||
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
|
||||
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
|
||||
directoryContent = filteredContent;
|
||||
}
|
||||
|
||||
return (directoryContent.Select(file =>
|
||||
{
|
||||
// we need to convert '\' into '/' because this path would be a part of a drop url
|
||||
var remoteFileName = I($"{dropPath}/{GetRelativePath(directoryPath, file.FileName).Replace('\\', '/')}");
|
||||
|
||||
return new DropItemForBuildXLFile(
|
||||
daemon.ApiClient,
|
||||
file.FileName,
|
||||
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
|
||||
conf.Get(EnableChunkDedup),
|
||||
file.ContentInfo,
|
||||
remoteFileName);
|
||||
}).ToArray(), null);
|
||||
}
|
||||
|
||||
private static string GetRelativePath(string root, string file)
|
||||
{
|
||||
var rootEndsWithSlash =
|
||||
root[root.Length - 1] == System.IO.Path.DirectorySeparatorChar
|
||||
|| root[root.Length - 1] == System.IO.Path.AltDirectorySeparatorChar;
|
||||
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
|
||||
}
|
||||
|
||||
private static async Task<(IEnumerable<DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
|
||||
ConfiguredCommand conf,
|
||||
DropDaemon daemon,
|
||||
string[] directoryPaths,
|
||||
string[] directoryIds,
|
||||
string[] dropPaths,
|
||||
Regex[] contentFilters)
|
||||
{
|
||||
Contract.Requires(directoryPaths != null);
|
||||
Contract.Requires(directoryIds != null);
|
||||
Contract.Requires(dropPaths != null);
|
||||
Contract.Requires(contentFilters != null);
|
||||
Contract.Requires(directoryPaths.Length == directoryIds.Length);
|
||||
Contract.Requires(directoryPaths.Length == dropPaths.Length);
|
||||
Contract.Requires(directoryPaths.Length == contentFilters.Length);
|
||||
|
||||
var createDropItemsTasks = Enumerable
|
||||
.Range(0, directoryPaths.Length)
|
||||
.Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();
|
||||
|
||||
var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);
|
||||
|
||||
if (createDropItemsResults.Any(r => r.error != null))
|
||||
{
|
||||
return (null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
|
||||
}
|
||||
|
||||
return (createDropItemsResults.SelectMany(r => r.Item1), null);
|
||||
}
|
||||
|
||||
private static (IEnumerable<DropItemForBuildXLFile>, string error) DedupeDropItems(IEnumerable<DropItemForBuildXLFile> dropItems)
|
||||
{
|
||||
var dropItemsByDropPaths = new Dictionary<string, DropItemForBuildXLFile>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var dropItem in dropItems)
|
||||
{
|
||||
if (dropItemsByDropPaths.TryGetValue(dropItem.RelativeDropPath, out var existingDropItem))
|
||||
{
|
||||
if (!string.Equals(dropItem.FullFilePath, existingDropItem.FullFilePath, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return (
|
||||
null,
|
||||
I($"'{dropItem.FullFilePath}' cannot be added to drop because it has the same drop path '{dropItem.RelativeDropPath}' as '{existingDropItem.FullFilePath}'"));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
dropItemsByDropPaths.Add(dropItem.RelativeDropPath, dropItem);
|
||||
}
|
||||
}
|
||||
|
||||
return (dropItemsByDropPaths.Select(kvp => kvp.Value).ToArray(), null);
|
||||
}
|
||||
|
||||
private static async Task<IIpcResult> AddDropItemsAsync(DropDaemon daemon, IEnumerable<DropItemForBuildXLFile> dropItems)
|
||||
{
|
||||
(IEnumerable<DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems);
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, error);
|
||||
}
|
||||
|
||||
var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray();
|
||||
var ipcResults = await TaskUtilities.SafeWhenAll(ipcResultTasks);
|
||||
|
||||
return IpcResult.Merge(ipcResults);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -10,6 +10,7 @@ using BuildXL.Cache.ContentStore.Hashing;
|
|||
using BuildXL.Ipc.ExternalApi;
|
||||
using BuildXL.Storage;
|
||||
using BuildXL.Utilities;
|
||||
using Tool.ServicePipDaemon;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
|
@ -69,29 +70,29 @@ namespace Tool.DropDaemon
|
|||
/// <summary>
|
||||
/// FileInfo is not already computed, sends an IPC request to BuildXL to materialize the file;
|
||||
/// if the request succeeds, returns a <see cref="FileInfo"/> corresponding to that file,
|
||||
/// otherwise throws a <see cref="DropDaemonException"/>.
|
||||
/// otherwise throws a <see cref="DaemonException"/>.
|
||||
/// </summary>
|
||||
public override async Task<FileInfo> EnsureMaterialized()
|
||||
{
|
||||
Possible<bool> maybeResult = await m_client.MaterializeFile(m_file, FullFilePath);
|
||||
if (!maybeResult.Succeeded)
|
||||
{
|
||||
throw new DropDaemonException(maybeResult.Failure.Describe());
|
||||
throw new DaemonException(maybeResult.Failure.Describe());
|
||||
}
|
||||
|
||||
if (!maybeResult.Result)
|
||||
{
|
||||
throw new DropDaemonException("File materialization failed");
|
||||
throw new DaemonException("File materialization failed");
|
||||
}
|
||||
|
||||
if (!File.Exists(FullFilePath))
|
||||
{
|
||||
throw new DropDaemonException("File materialization succeeded, but file is not found on disk: " + FullFilePath);
|
||||
throw new DaemonException("File materialization succeeded, but file is not found on disk: " + FullFilePath);
|
||||
}
|
||||
|
||||
if (m_symlinkTester(FullFilePath))
|
||||
{
|
||||
throw new DropDaemonException(MaterializationResultIsSymlinkErrorPrefix + FullFilePath);
|
||||
throw new DaemonException(MaterializationResultIsSymlinkErrorPrefix + FullFilePath);
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
|
|
|
@ -7,6 +7,7 @@ using System.Threading;
|
|||
using System.Threading.Tasks;
|
||||
using BuildXL.Storage;
|
||||
using Microsoft.VisualStudio.Services.BlobStore.Common;
|
||||
using Tool.ServicePipDaemon;
|
||||
using static BuildXL.Utilities.FormattableStringEx;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
|
@ -75,7 +76,7 @@ namespace Tool.DropDaemon
|
|||
{
|
||||
if (!File.Exists(FullFilePath))
|
||||
{
|
||||
throw new DropDaemonException("File not found on disk: " + FullFilePath);
|
||||
throw new DaemonException("File not found on disk: " + FullFilePath);
|
||||
}
|
||||
|
||||
return Task.FromResult(new FileInfo(FullFilePath));
|
||||
|
@ -97,7 +98,7 @@ namespace Tool.DropDaemon
|
|||
|
||||
/// <summary>
|
||||
/// Checks if a given <see cref="BlobIdentifier"/> matches one computed from a file on disk
|
||||
/// (at <paramref name="filePath"/> location). If it doesn't, throws <see cref="DropDaemonException"/>.
|
||||
/// (at <paramref name="filePath"/> location). If it doesn't, throws <see cref="DaemonException"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This method should only be called from '#if DEBUG' blocks, because it's wasteful to recompute hashes all the time.
|
||||
|
@ -111,20 +112,20 @@ namespace Tool.DropDaemon
|
|||
var calculated = (await ComputeFileDescriptorFromFileAsync(filePath, chunkDedup, Path.GetFileName(filePath), cancellationToken)).BlobIdentifier;
|
||||
if (!precomputed.Equals(calculated))
|
||||
{
|
||||
throw new DropDaemonException(I($"[{phase}] Given blob identifier ({precomputed}) differs from computed one ({calculated}) for file '{filePath}'."));
|
||||
throw new DaemonException(I($"[{phase}] Given blob identifier ({precomputed}) differs from computed one ({calculated}) for file '{filePath}'."));
|
||||
}
|
||||
|
||||
var actualFileLength = new FileInfo(filePath).Length;
|
||||
if (expectedFileLength > 0 && expectedFileLength != actualFileLength)
|
||||
{
|
||||
throw new DropDaemonException(I($"[{phase}] Given file length ({expectedFileLength}) differs from the file size on disk({actualFileLength}) for file '{filePath}'."));
|
||||
throw new DaemonException(I($"[{phase}] Given file length ({expectedFileLength}) differs from the file size on disk({actualFileLength}) for file '{filePath}'."));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deserializes <see cref="BlobIdentifier"/> from a string.
|
||||
///
|
||||
/// Throws <see cref="DropDaemonException"/> if the string cannot be parsed.
|
||||
/// Throws <see cref="DaemonException"/> if the string cannot be parsed.
|
||||
/// </summary>
|
||||
private static BlobIdentifier DeserializeBlobIdentifierFromHash(string serializedVsoHash)
|
||||
{
|
||||
|
@ -134,7 +135,7 @@ namespace Tool.DropDaemon
|
|||
BuildXL.Cache.ContentStore.Hashing.ContentHash contentHash;
|
||||
if (!BuildXL.Cache.ContentStore.Hashing.ContentHash.TryParse(serializedVsoHash, out contentHash))
|
||||
{
|
||||
throw new DropDaemonException("Could not parse content hash: " + serializedVsoHash);
|
||||
throw new DaemonException("Could not parse content hash: " + serializedVsoHash);
|
||||
}
|
||||
|
||||
return new BlobIdentifier(contentHash.ToHashByteArray());
|
||||
|
|
|
@ -10,43 +10,43 @@ using Microsoft.VisualStudio.Services.Drop.WebApi;
|
|||
namespace Tool.DropDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Abstraction for communicating with a drop service endpoint.
|
||||
/// Abstraction for communicating with a drop service endpoint.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// All the the methods/properties in this interface assume that the concrete <see cref="IDropClient"/> instance
|
||||
/// has already been initialized with the necessary drop settings (<see cref="DropConfig"/>).
|
||||
/// All the methods/properties in this interface assume that the concrete <see cref="IDropClient"/> instance
|
||||
/// has already been initialized with the necessary drop settings (<see cref="DropConfig"/>).
|
||||
/// </remarks>
|
||||
public interface IDropClient : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// URL at which the drop can be obtained/viewed.
|
||||
/// URL at which the drop can be obtained/viewed.
|
||||
/// </summary>
|
||||
string DropUrl { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Task for performing 'drop create'.
|
||||
/// Task for performing 'drop create'.
|
||||
/// </summary>
|
||||
Task<DropItem> CreateAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Task for performing 'drop addfile'.
|
||||
/// Task for performing 'drop addfile'.
|
||||
/// </summary>
|
||||
Task<AddFileResult> AddFileAsync([NotNull]IDropItem dropItem);
|
||||
|
||||
/// <summary>
|
||||
/// Task for performing 'drop finalize'.
|
||||
/// Task for performing 'drop finalize'.
|
||||
/// </summary>
|
||||
Task<FinalizeResult> FinalizeAsync();
|
||||
|
||||
/// <summary>
|
||||
/// Arbitrary statistics to report;
|
||||
/// Arbitrary statistics to report;
|
||||
/// </summary>
|
||||
[NotNull]
|
||||
IDictionary<string, long> GetStats();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of the 'AddFile' operation (called on a single file).
|
||||
/// Result of the 'AddFile' operation (called on a single file).
|
||||
/// </summary>
|
||||
public enum AddFileResult
|
||||
{
|
||||
|
@ -58,7 +58,7 @@ namespace Tool.DropDaemon
|
|||
}
|
||||
|
||||
/// <summary>
|
||||
/// Placeholder for future result that might be returned by <see cref="IDropClient.FinalizeAsync"/>.
|
||||
/// Placeholder for future result that might be returned by <see cref="IDropClient.FinalizeAsync"/>.
|
||||
/// </summary>
|
||||
public sealed class FinalizeResult
|
||||
{ }
|
||||
|
|
|
@ -11,42 +11,42 @@ using Pure = System.Diagnostics.Contracts.PureAttribute;
|
|||
namespace Tool.DropDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Item to be added to drop.
|
||||
/// Item to be added to drop.
|
||||
/// </summary>
|
||||
public interface IDropItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Full path of the file on disk to be added to drop. The file need not be
|
||||
/// physically present on disk before <see cref="EnsureMaterialized"/> is called.
|
||||
/// Full path of the file on disk to be added to drop. The file need not be
|
||||
/// physically present on disk before <see cref="EnsureMaterialized"/> is called.
|
||||
/// </summary>
|
||||
[NotNull]
|
||||
[Pure]
|
||||
string FullFilePath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Relative path under which to associate the file with a drop.
|
||||
/// Relative path under which to associate the file with a drop.
|
||||
/// </summary>
|
||||
[NotNull]
|
||||
[Pure]
|
||||
string RelativeDropPath { get; }
|
||||
|
||||
/// <summary>
|
||||
/// (Optional) Pre-computed blob identifier.
|
||||
/// (Optional) Pre-computed blob identifier.
|
||||
/// </summary>
|
||||
[CanBeNull]
|
||||
[Pure]
|
||||
BlobIdentifier BlobIdentifier { get; }
|
||||
|
||||
/// <summary>
|
||||
/// (Optional) Pre-computed file length.
|
||||
/// (Optional) Pre-computed file length.
|
||||
/// </summary>
|
||||
[Pure]
|
||||
long FileLength { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Full information about the file which is to be added to a drop. After the completion
|
||||
/// of the returned task, the file must exist on disk. The returned file info must also
|
||||
/// match the full file path returned by the <see cref="FullFilePath"/> property.
|
||||
/// Full information about the file which is to be added to a drop. After the completion
|
||||
/// of the returned task, the file must exist on disk. The returned file info must also
|
||||
/// match the full file path returned by the <see cref="FullFilePath"/> property.
|
||||
/// </summary>
|
||||
[NotNull]
|
||||
Task<FileInfo> EnsureMaterialized();
|
||||
|
|
|
@ -2,653 +2,21 @@
|
|||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.ExternalApi;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Scheduler;
|
||||
using BuildXL.Storage;
|
||||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.CLI;
|
||||
using BuildXL.Utilities.Tasks;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using static Tool.DropDaemon.Statics;
|
||||
using Tool.ServicePipDaemon;
|
||||
using static Tool.ServicePipDaemon.Statics;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// DropDaemon entry point.
|
||||
/// DropDaemon entry point.
|
||||
/// </summary>
|
||||
public static class Program
|
||||
{
|
||||
internal static IIpcProvider IpcProvider = IpcFactory.GetProvider();
|
||||
internal static readonly List<Option> DaemonConfigOptions = new List<Option>();
|
||||
internal static readonly List<Option> DropConfigOptions = new List<Option>();
|
||||
|
||||
private const int ServicePointParallelismForDrop = 200;
|
||||
private static readonly int s_minIoThreadsForDrop = Environment.ProcessorCount * 10;
|
||||
private static readonly int s_minWorkerThreadsForDrop = Environment.ProcessorCount * 10;
|
||||
|
||||
private const string IncludeAllFilter = ".*";
|
||||
|
||||
// ==============================================================================
|
||||
// Daemon config
|
||||
// ==============================================================================
|
||||
internal static readonly StrOption Moniker = RegisterDaemonConfigOption(new StrOption("moniker")
|
||||
{
|
||||
ShortName = "m",
|
||||
HelpText = "Moniker to identify client/server communication",
|
||||
});
|
||||
|
||||
internal static readonly IntOption MaxConcurrentClients = RegisterDaemonConfigOption(new IntOption("maxConcurrentClients")
|
||||
{
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Maximum number of clients to serve concurrently)",
|
||||
DefaultValue = DaemonConfig.DefaultMaxConcurrentClients,
|
||||
});
|
||||
|
||||
internal static readonly IntOption MaxConnectRetries = RegisterDaemonConfigOption(new IntOption("maxConnectRetries")
|
||||
{
|
||||
HelpText = "Maximum number of retries to establish a connection with a running daemon",
|
||||
DefaultValue = DaemonConfig.DefaultMaxConnectRetries,
|
||||
});
|
||||
|
||||
internal static readonly IntOption ConnectRetryDelayMillis = RegisterDaemonConfigOption(new IntOption("connectRetryDelayMillis")
|
||||
{
|
||||
HelpText = "Delay between consecutive retries to establish a connection with a running daemon",
|
||||
DefaultValue = (int)DaemonConfig.DefaultConnectRetryDelay.TotalMilliseconds,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption ShellExecute = RegisterDaemonConfigOption(new BoolOption("shellExecute")
|
||||
{
|
||||
HelpText = "Use shell execute to start the daemon process (a shell window will be created and displayed)",
|
||||
DefaultValue = false,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption StopOnFirstFailure = RegisterDaemonConfigOption(new BoolOption("stopOnFirstFailure")
|
||||
{
|
||||
HelpText = "Daemon process should terminate after first failed operation (e.g., 'drop create' fails because the drop already exists).",
|
||||
DefaultValue = DaemonConfig.DefaultStopOnFirstFailure,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption EnableCloudBuildIntegration = RegisterDaemonConfigOption(new BoolOption("enableCloudBuildIntegration")
|
||||
{
|
||||
ShortName = "ecb",
|
||||
HelpText = "Enable logging ETW events for CloudBuild to pick up",
|
||||
DefaultValue = DaemonConfig.DefaultEnableCloudBuildIntegration,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption Verbose = RegisterDaemonConfigOption(new BoolOption("verbose")
|
||||
{
|
||||
ShortName = "v",
|
||||
HelpText = "Verbose logging",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultVerbose,
|
||||
});
|
||||
|
||||
internal static readonly StrOption DropServiceConfigFile = RegisterDaemonConfigOption(new StrOption("dropServiceConfigFile")
|
||||
{
|
||||
ShortName = "c",
|
||||
HelpText = "Drop service configuration file",
|
||||
DefaultValue = null,
|
||||
Expander = (fileName) =>
|
||||
{
|
||||
var json = System.IO.File.ReadAllText(fileName);
|
||||
var jObject = JObject.Parse(json);
|
||||
return jObject.Properties().Select(prop => new ParsedOption(PrefixKind.Long, prop.Name, prop.Value.ToString()));
|
||||
},
|
||||
});
|
||||
|
||||
internal static readonly StrOption LogDir = RegisterDaemonConfigOption(new StrOption("logDir")
|
||||
{
|
||||
ShortName = "log",
|
||||
HelpText = "Log directory",
|
||||
IsRequired = false
|
||||
});
|
||||
|
||||
// ==============================================================================
|
||||
// Drop config
|
||||
// ==============================================================================
|
||||
internal static readonly StrOption DropName = RegisterDropConfigOption(new StrOption("name")
|
||||
{
|
||||
ShortName = "n",
|
||||
HelpText = "Drop name",
|
||||
IsRequired = true,
|
||||
});
|
||||
|
||||
internal static readonly UriOption DropEndpoint = RegisterDropConfigOption(new UriOption("service")
|
||||
{
|
||||
ShortName = "s",
|
||||
HelpText = "Drop endpoint URI",
|
||||
IsRequired = true,
|
||||
});
|
||||
|
||||
internal static readonly IntOption BatchSize = RegisterDropConfigOption(new IntOption("batchSize")
|
||||
{
|
||||
ShortName = "bs",
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Size of batches in which to send 'associate' requests)",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultBatchSizeForAssociate,
|
||||
});
|
||||
|
||||
internal static readonly IntOption MaxParallelUploads = RegisterDropConfigOption(new IntOption("maxParallelUploads")
|
||||
{
|
||||
ShortName = "mpu",
|
||||
HelpText = "Maximum number of uploads to issue to drop service in parallel",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultMaxParallelUploads,
|
||||
});
|
||||
|
||||
internal static readonly IntOption NagleTimeMillis = RegisterDropConfigOption(new IntOption("nagleTimeMillis")
|
||||
{
|
||||
ShortName = "nt",
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Maximum time in milliseconds to wait before triggering a batch 'associate' request)",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultNagleTimeForAssociate.TotalMilliseconds,
|
||||
});
|
||||
|
||||
internal static readonly IntOption RetentionDays = RegisterDropConfigOption(new IntOption("retentionDays")
|
||||
{
|
||||
ShortName = "rt",
|
||||
HelpText = "Drop retention time in days",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultRetention.TotalDays,
|
||||
});
|
||||
|
||||
internal static readonly IntOption HttpSendTimeoutMillis = RegisterDropConfigOption(new IntOption("httpSendTimeoutMillis")
|
||||
{
|
||||
HelpText = "Timeout for http requests",
|
||||
IsRequired = false,
|
||||
DefaultValue = (int)DropConfig.DefaultHttpSendTimeout.TotalMilliseconds,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption EnableTelemetry = RegisterDropConfigOption(new BoolOption("enableTelemetry")
|
||||
{
|
||||
ShortName = "t",
|
||||
HelpText = "Verbose logging",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultEnableTelemetry,
|
||||
});
|
||||
|
||||
internal static readonly BoolOption EnableChunkDedup = RegisterDropConfigOption(new BoolOption("enableChunkDedup")
|
||||
{
|
||||
ShortName = "cd",
|
||||
HelpText = "Chunk level dedup",
|
||||
IsRequired = false,
|
||||
DefaultValue = DropConfig.DefaultEnableChunkDedup,
|
||||
});
|
||||
|
||||
// ==============================================================================
|
||||
// 'addfile' and 'addartifacts' parameters
|
||||
// ==============================================================================
|
||||
internal static readonly StrOption RelativeDropPath = new StrOption("dropPath")
|
||||
{
|
||||
ShortName = "d",
|
||||
HelpText = "Relative drop path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption File = new StrOption("file")
|
||||
{
|
||||
ShortName = "f",
|
||||
HelpText = "File path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption HashOptional = new StrOption("hash")
|
||||
{
|
||||
ShortName = "h",
|
||||
HelpText = "VSO file hash",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption FileId = new StrOption("fileId")
|
||||
{
|
||||
ShortName = "fid",
|
||||
HelpText = "BuildXL file identifier",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption IpcServerMonikerRequired = new StrOption("ipcServerMoniker")
|
||||
{
|
||||
ShortName = "dm",
|
||||
HelpText = "IPC moniker identifying a running BuildXL IPC server",
|
||||
IsRequired = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption HelpNoNameOption = new StrOption(string.Empty)
|
||||
{
|
||||
HelpText = "Command name",
|
||||
};
|
||||
|
||||
internal static readonly StrOption Directory = new StrOption("directory")
|
||||
{
|
||||
ShortName = "dir",
|
||||
HelpText = "Directory path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption DirectoryId = new StrOption("directoryId")
|
||||
{
|
||||
ShortName = "dirid",
|
||||
HelpText = "BuildXL directory identifier",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption RelativeDirectoryDropPath = new StrOption("directoryDropPath")
|
||||
{
|
||||
ShortName = "dird",
|
||||
HelpText = "Relative drop path for directory",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
internal static readonly StrOption DirectoryContentFilter = new StrOption("directoryFilter")
|
||||
{
|
||||
ShortName = "dcfilter",
|
||||
HelpText = "Directory content filter (only files that match the filter will be added to drop).",
|
||||
DefaultValue = null,
|
||||
IsRequired = false,
|
||||
IsMultiValue = false,
|
||||
};
|
||||
|
||||
// ==============================================================================
|
||||
// Commands
|
||||
// ==============================================================================
|
||||
internal static readonly Dictionary<string, Command> Commands = new Dictionary<string, Command>();
|
||||
|
||||
/// <remarks>
|
||||
/// The <see cref="DaemonConfigOptions"/> options are added to every command.
|
||||
/// A non-mandatory string option "name" is added as well, which drop operation
|
||||
/// commands may want to use to explicitly specify the target drop name.
|
||||
/// </remarks>
|
||||
private static Command RegisterCommand(
|
||||
string name,
|
||||
IEnumerable<Option> options = null,
|
||||
ServerAction serverAction = null,
|
||||
ClientAction clientAction = null,
|
||||
string description = null,
|
||||
bool needsIpcClient = true,
|
||||
bool addDaemonConfigOptions = true)
|
||||
{
|
||||
var opts = (options ?? new Option[0]).ToList();
|
||||
if (addDaemonConfigOptions)
|
||||
{
|
||||
opts.AddRange(DaemonConfigOptions);
|
||||
}
|
||||
|
||||
if (!opts.Exists(opt => opt.LongName == "name"))
|
||||
{
|
||||
opts.Add(new Option(longName: DropName.LongName)
|
||||
{
|
||||
ShortName = DropName.ShortName,
|
||||
});
|
||||
}
|
||||
|
||||
var cmd = new Command(name, opts, serverAction, clientAction, description, needsIpcClient);
|
||||
Commands[cmd.Name] = cmd;
|
||||
return cmd;
|
||||
}
|
||||
|
||||
internal static readonly Command HelpCmd = RegisterCommand(
|
||||
name: "help",
|
||||
description: "Prints a help message (usage).",
|
||||
options: new[] { HelpNoNameOption },
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, rpc) =>
|
||||
{
|
||||
string cmdName = conf.Get(HelpNoNameOption);
|
||||
bool cmdNotSpecified = string.IsNullOrWhiteSpace(cmdName);
|
||||
if (cmdNotSpecified)
|
||||
{
|
||||
Console.WriteLine(Usage());
|
||||
return 0;
|
||||
}
|
||||
|
||||
Command requestedHelpForCommand;
|
||||
var requestedCommandFound = Commands.TryGetValue(cmdName, out requestedHelpForCommand);
|
||||
if (requestedCommandFound)
|
||||
{
|
||||
Console.WriteLine(requestedHelpForCommand.Usage(conf.Config.Parser));
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine(Usage());
|
||||
return 1;
|
||||
}
|
||||
});
|
||||
|
||||
internal static readonly Command StartNoDropCmd = RegisterCommand(
|
||||
name: "start-nodrop",
|
||||
description: @"Starts a server process without a backing VSO drop client (useful for testing/pinging the daemon).",
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
var dropConfig = new DropConfig(string.Empty, new Uri("file://xyz"));
|
||||
var daemonConfig = CreateDaemonConfig(conf);
|
||||
var vsoClientTask = TaskSourceSlim.Create<IDropClient>();
|
||||
vsoClientTask.SetException(new NotSupportedException());
|
||||
using (var daemon = new Daemon(conf.Config.Parser, daemonConfig, dropConfig, vsoClientTask.Task))
|
||||
{
|
||||
daemon.Start();
|
||||
daemon.Completion.GetAwaiter().GetResult();
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
internal static readonly StrOption IpcServerMonikerOptional = new StrOption(
|
||||
longName: IpcServerMonikerRequired.LongName)
|
||||
{
|
||||
ShortName = IpcServerMonikerRequired.ShortName,
|
||||
HelpText = IpcServerMonikerRequired.HelpText,
|
||||
IsRequired = false,
|
||||
};
|
||||
|
||||
internal static readonly Command StartCmd = RegisterCommand(
|
||||
name: "start",
|
||||
description: "Starts the server process.",
|
||||
options: DropConfigOptions.Union(new[] { IpcServerMonikerOptional }),
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
SetupThreadPoolAndServicePoint();
|
||||
var dropConfig = CreateDropConfig(conf);
|
||||
var daemonConf = CreateDaemonConfig(conf);
|
||||
|
||||
if (daemonConf.MaxConcurrentClients <= 1)
|
||||
{
|
||||
conf.Logger.Error($"Must specify at least 2 '{nameof(DaemonConfig.MaxConcurrentClients)}' when running DropDaemon to avoid deadlock when stopping this daemon from a different client");
|
||||
return -1;
|
||||
}
|
||||
|
||||
using (var client = CreateClient(conf.Get(IpcServerMonikerOptional), daemonConf))
|
||||
using (var daemon = new Daemon(
|
||||
parser: conf.Config.Parser,
|
||||
daemonConfig: daemonConf,
|
||||
dropConfig: dropConfig,
|
||||
dropClientTask: null,
|
||||
client: client))
|
||||
{
|
||||
daemon.Start();
|
||||
daemon.Completion.GetAwaiter().GetResult();
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
|
||||
private static void SetupThreadPoolAndServicePoint()
|
||||
{
|
||||
int workerThreads, ioThreads;
|
||||
ThreadPool.GetMinThreads(out workerThreads, out ioThreads);
|
||||
|
||||
workerThreads = Math.Max(workerThreads, s_minWorkerThreadsForDrop);
|
||||
ioThreads = Math.Max(ioThreads, s_minIoThreadsForDrop);
|
||||
ThreadPool.SetMinThreads(workerThreads, ioThreads);
|
||||
|
||||
ServicePointManager.DefaultConnectionLimit = Math.Max(ServicePointParallelismForDrop, ServicePointManager.DefaultConnectionLimit);
|
||||
}
|
||||
|
||||
internal static readonly Command StartDaemonCmd = RegisterCommand(
|
||||
name: "start-daemon",
|
||||
description: "Starts the server process in background (as daemon).",
|
||||
options: DropConfigOptions,
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, _) =>
|
||||
{
|
||||
using (var daemon = new Process())
|
||||
{
|
||||
bool shellExecute = conf.Get(ShellExecute);
|
||||
daemon.StartInfo.FileName = AssemblyHelper.GetAssemblyLocation(System.Reflection.Assembly.GetEntryAssembly());
|
||||
daemon.StartInfo.Arguments = "start " + conf.Config.Render();
|
||||
daemon.StartInfo.LoadUserProfile = false;
|
||||
daemon.StartInfo.UseShellExecute = shellExecute;
|
||||
daemon.StartInfo.CreateNoWindow = !shellExecute;
|
||||
daemon.Start();
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
internal static readonly Command StopDaemonCmd = RegisterCommand(
|
||||
name: "stop",
|
||||
description: "[RPC] Stops the daemon process running on specified port; fails if no such daemon is running.",
|
||||
clientAction: AsyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
conf.Logger.Info("[STOP] requested");
|
||||
daemon.RequestStop();
|
||||
return Task.FromResult(IpcResult.Success());
|
||||
});
|
||||
|
||||
internal static readonly Command CrashDaemonCmd = RegisterCommand(
|
||||
name: "crash",
|
||||
description: "[RPC] Stops the server process by crashing it.",
|
||||
clientAction: AsyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[CRASH] requested");
|
||||
Environment.Exit(-1);
|
||||
return Task.FromResult(IpcResult.Success());
|
||||
});
|
||||
|
||||
internal static readonly Command PingDaemonCmd = RegisterCommand(
|
||||
name: "ping",
|
||||
description: "[RPC] Pings the daemon process.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[PING] received");
|
||||
return Task.FromResult(IpcResult.Success("Alive!"));
|
||||
});
|
||||
|
||||
internal static readonly Command CreateDropCmd = RegisterCommand(
|
||||
name: "create",
|
||||
description: "[RPC] Invokes the 'create' operation.",
|
||||
options: DropConfigOptions,
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[CREATE]: Started at " + daemon.DropConfig.Service + "/" + daemon.DropName);
|
||||
IIpcResult result = await daemon.CreateAsync();
|
||||
daemon.Logger.Info("[CREATE]: " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command FinalizeDropCmd = RegisterCommand(
|
||||
name: "finalize",
|
||||
description: "[RPC] Invokes the 'finalize' operation.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[FINALIZE] Started at" + daemon.DropConfig.Service + "/" + daemon.DropName);
|
||||
IIpcResult result = await daemon.FinalizeAsync();
|
||||
daemon.Logger.Info("[FINALIZE] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command FinalizeDropAndStopDaemonCmd = RegisterCommand(
|
||||
name: "finalize-and-stop",
|
||||
description: "[RPC] Invokes the 'finalize' operation; then stops the daemon.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: Command.Compose(FinalizeDropCmd.ServerAction, StopDaemonCmd.ServerAction));
|
||||
|
||||
internal static readonly Command AddFileToDropCmd = RegisterCommand(
|
||||
name: "addfile",
|
||||
description: "[RPC] invokes the 'addfile' operation.",
|
||||
options: new Option[] { File, RelativeDropPath, HashOptional },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Verbose("[ADDFILE] Started");
|
||||
string filePath = conf.Get(File);
|
||||
string hashValue = conf.Get(HashOptional);
|
||||
var contentInfo = string.IsNullOrEmpty(hashValue) ? null : (FileContentInfo?)FileContentInfo.Parse(hashValue);
|
||||
var dropItem = new DropItemForFile(filePath, conf.Get(RelativeDropPath), contentInfo);
|
||||
IIpcResult result = System.IO.File.Exists(filePath)
|
||||
? await daemon.AddFileAsync(dropItem)
|
||||
: new IpcResult(IpcResultStatus.ExecutionError, "file '" + filePath + "' does not exist");
|
||||
daemon.Logger.Verbose("[ADDFILE] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
internal static readonly Command AddArtifactsToDropCmd = RegisterCommand(
|
||||
name: "addartifacts",
|
||||
description: "[RPC] invokes the 'addartifacts' operation.",
|
||||
options: new Option[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath, Directory, DirectoryId, RelativeDirectoryDropPath, DirectoryContentFilter },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: async (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Verbose("[ADDARTIFACTS] Started");
|
||||
|
||||
var result = await AddArtifactsToDropInternalAsync(conf, daemon);
|
||||
|
||||
daemon.Logger.Verbose("[ADDARTIFACTS] " + result);
|
||||
return result;
|
||||
});
|
||||
|
||||
private static async Task<IIpcResult> AddArtifactsToDropInternalAsync(ConfiguredCommand conf, Daemon daemon)
|
||||
{
|
||||
var files = File.GetValues(conf.Config).ToArray();
|
||||
var fileIds = FileId.GetValues(conf.Config).ToArray();
|
||||
var hashes = HashOptional.GetValues(conf.Config).ToArray();
|
||||
var dropPaths = RelativeDropPath.GetValues(conf.Config).ToArray();
|
||||
|
||||
if (files.Length != fileIds.Length || files.Length != hashes.Length || files.Length != dropPaths.Length)
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.GenericError,
|
||||
Inv(
|
||||
"File counts don't match: #files = {0}, #fileIds = {1}, #hashes = {2}, #dropPaths = {3}",
|
||||
files.Length, fileIds.Length, hashes.Length, dropPaths.Length));
|
||||
}
|
||||
|
||||
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
|
||||
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
|
||||
var directoryDropPaths = RelativeDirectoryDropPath.GetValues(conf.Config).ToArray();
|
||||
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
|
||||
|
||||
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryDropPaths.Length || directoryPaths.Length != directoryFilters.Length)
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.GenericError,
|
||||
Inv(
|
||||
"Directory counts don't match: #directories = {0}, #directoryIds = {1}, #dropPaths = {2}, #directoryFilters = {3}",
|
||||
directoryPaths.Length, directoryIds.Length, directoryDropPaths.Length, directoryFilters.Length));
|
||||
}
|
||||
|
||||
(Regex[] initializedFilters, string filterInitError) = InitializeDirectoryFilters(directoryFilters);
|
||||
if (filterInitError != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, filterInitError);
|
||||
}
|
||||
|
||||
var dropFileItemsKeyedByIsAbsent = Enumerable
|
||||
.Range(0, files.Length)
|
||||
.Select(i => new DropItemForBuildXLFile(
|
||||
daemon.ApiClient,
|
||||
chunkDedup: conf.Get(EnableChunkDedup),
|
||||
filePath: files[i],
|
||||
fileId: fileIds[i],
|
||||
fileContentInfo: FileContentInfo.Parse(hashes[i]),
|
||||
relativeDropPath: dropPaths[i])).ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
|
||||
|
||||
// If a user specified a particular file to be added to drop, this file must be a part of drop.
|
||||
// The missing files will not get into the drop, so we emit an error.
|
||||
if (dropFileItemsKeyedByIsAbsent[true].Any())
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.InvalidInput,
|
||||
Inv("The following files are missing, but they are a part of the drop command:{0}{1}",
|
||||
Environment.NewLine,
|
||||
string.Join(Environment.NewLine, dropFileItemsKeyedByIsAbsent[true])));
|
||||
}
|
||||
|
||||
(IEnumerable<DropItemForBuildXLFile> dropDirectoryMemberItems, string error) = await CreateDropItemsForDirectoriesAsync(
|
||||
conf,
|
||||
daemon,
|
||||
directoryPaths,
|
||||
directoryIds,
|
||||
directoryDropPaths,
|
||||
initializedFilters);
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, error);
|
||||
}
|
||||
|
||||
var groupedDirectoriesContent = dropDirectoryMemberItems.ToLookup(f => WellKnownContentHashUtilities.IsAbsentFileHash(f.Hash));
|
||||
|
||||
// we allow missing files inside of directories only if those files are output files (e.g., optional or temporary files)
|
||||
if (groupedDirectoriesContent[true].Any(f => !f.IsOutputFile))
|
||||
{
|
||||
return new IpcResult(
|
||||
IpcResultStatus.InvalidInput,
|
||||
Inv("Uploading missing source file(s) is not supported:{0}{1}",
|
||||
Environment.NewLine,
|
||||
string.Join(Environment.NewLine, groupedDirectoriesContent[true].Where(f => !f.IsOutputFile))));
|
||||
}
|
||||
|
||||
// return early if there is nothing to upload
|
||||
if (!dropFileItemsKeyedByIsAbsent[false].Any() && !groupedDirectoriesContent[false].Any())
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.Success, string.Empty);
|
||||
}
|
||||
|
||||
return await AddDropItemsAsync(daemon, dropFileItemsKeyedByIsAbsent[false].Concat(groupedDirectoriesContent[false]));
|
||||
}
|
||||
|
||||
private static (Regex[], string error) InitializeDirectoryFilters(string[] filters)
|
||||
{
|
||||
try
|
||||
{
|
||||
var initializedFilters = filters.Select(
|
||||
filter => filter == IncludeAllFilter
|
||||
? null
|
||||
: new Regex(filter, RegexOptions.Compiled | RegexOptions.IgnoreCase));
|
||||
|
||||
return (initializedFilters.ToArray(), null);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return (null, e.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
internal static readonly Command TestReadFile = RegisterCommand(
|
||||
name: "test-readfile",
|
||||
description: "[RPC] Sends a request to the daemon to read a file.",
|
||||
options: new Option[] { File },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[READFILE] received");
|
||||
var result = IpcResult.Success(System.IO.File.ReadAllText(conf.Get(File)));
|
||||
daemon.Logger.Info("[READFILE] succeeded");
|
||||
return Task.FromResult(result);
|
||||
});
|
||||
|
||||
private static Client CreateClient(string serverMoniker, IClientConfig config)
|
||||
{
|
||||
return serverMoniker != null
|
||||
? Client.Create(serverMoniker, config)
|
||||
: null;
|
||||
}
|
||||
|
||||
/// <nodoc/>
|
||||
[SuppressMessage("Microsoft.Naming", "CA2204:Spelling of DropD")]
|
||||
public static int Main(string[] args)
|
||||
|
@ -658,17 +26,19 @@ namespace Tool.DropDaemon
|
|||
|
||||
try
|
||||
{
|
||||
Console.WriteLine("DropDaemon started at " + DateTime.UtcNow);
|
||||
Console.WriteLine(Daemon.DropDLogPrefix + "Command line arguments: ");
|
||||
Console.WriteLine(string.Join(Environment.NewLine + Daemon.DropDLogPrefix, args));
|
||||
Console.WriteLine(nameof(DropDaemon) + " started at " + DateTime.UtcNow);
|
||||
Console.WriteLine(DropDaemon.DropDLogPrefix + "Command line arguments: ");
|
||||
Console.WriteLine(string.Join(Environment.NewLine + DropDaemon.DropDLogPrefix, args));
|
||||
Console.WriteLine();
|
||||
|
||||
ConfiguredCommand conf = ParseArgs(args, new UnixParser());
|
||||
if (conf.Command.NeedsIpcClient)
|
||||
DropDaemon.EnsureCommandsInitialized();
|
||||
|
||||
var confCommand = ServicePipDaemon.ServicePipDaemon.ParseArgs(args, new UnixParser());
|
||||
if (confCommand.Command.NeedsIpcClient)
|
||||
{
|
||||
using (var rpc = CreateClient(conf))
|
||||
using (var rpc = CreateClient(confCommand))
|
||||
{
|
||||
var result = conf.Command.ClientAction(conf, rpc);
|
||||
var result = confCommand.Command.ClientAction(confCommand, rpc);
|
||||
rpc.RequestStop();
|
||||
rpc.Completion.GetAwaiter().GetResult();
|
||||
return result;
|
||||
|
@ -676,7 +46,7 @@ namespace Tool.DropDaemon
|
|||
}
|
||||
else
|
||||
{
|
||||
return conf.Command.ClientAction(conf, null);
|
||||
return confCommand.Command.ClientAction(confCommand, null);
|
||||
}
|
||||
}
|
||||
catch (ArgumentException e)
|
||||
|
@ -686,258 +56,10 @@ namespace Tool.DropDaemon
|
|||
}
|
||||
}
|
||||
|
||||
internal static ConfiguredCommand ParseArgs(string allArgs, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
|
||||
{
|
||||
return ParseArgs(parser.SplitArgs(allArgs), parser, logger, ignoreInvalidOptions);
|
||||
}
|
||||
|
||||
internal static ConfiguredCommand ParseArgs(string[] args, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
|
||||
{
|
||||
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
|
||||
|
||||
if (args.Length == 0)
|
||||
{
|
||||
throw new ArgumentException(Inv("Command is required. {0}", usageMessage.Value));
|
||||
}
|
||||
|
||||
var argsQueue = new Queue<string>(args);
|
||||
string cmdName = argsQueue.Dequeue();
|
||||
if (!Commands.TryGetValue(cmdName, out Command cmd))
|
||||
{
|
||||
throw new ArgumentException(Inv("No command '{0}' is found. {1}", cmdName, usageMessage.Value));
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
Config conf = Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
|
||||
var parseTime = sw.Elapsed;
|
||||
|
||||
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), Daemon.DropDLogPrefix);
|
||||
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
|
||||
return new ConfiguredCommand(cmd, conf, logger);
|
||||
}
|
||||
|
||||
internal static DropConfig CreateDropConfig(ConfiguredCommand conf)
|
||||
{
|
||||
return new DropConfig(
|
||||
dropName: conf.Get(DropName),
|
||||
serviceEndpoint: conf.Get(DropEndpoint),
|
||||
maxParallelUploads: conf.Get(MaxParallelUploads),
|
||||
retention: TimeSpan.FromDays(conf.Get(RetentionDays)),
|
||||
httpSendTimeout: TimeSpan.FromMilliseconds(conf.Get(HttpSendTimeoutMillis)),
|
||||
verbose: conf.Get(Verbose),
|
||||
enableTelemetry: conf.Get(EnableTelemetry),
|
||||
enableChunkDedup: conf.Get(EnableChunkDedup),
|
||||
logDir: conf.Get(LogDir));
|
||||
}
|
||||
|
||||
internal static DaemonConfig CreateDaemonConfig(ConfiguredCommand conf)
|
||||
{
|
||||
return new DaemonConfig(
|
||||
logger: conf.Logger,
|
||||
moniker: conf.Get(Moniker),
|
||||
maxConnectRetries: conf.Get(MaxConnectRetries),
|
||||
connectRetryDelay: TimeSpan.FromMilliseconds(conf.Get(ConnectRetryDelayMillis)),
|
||||
stopOnFirstFailure: conf.Get(StopOnFirstFailure),
|
||||
enableCloudBuildIntegration: conf.Get(EnableCloudBuildIntegration));
|
||||
}
|
||||
|
||||
internal static IClient CreateClient(ConfiguredCommand conf)
|
||||
{
|
||||
var daemonConfig = CreateDaemonConfig(conf);
|
||||
return IpcProvider.GetClient(daemonConfig.Moniker, daemonConfig);
|
||||
var daemonConfig = ServicePipDaemon.ServicePipDaemon.CreateDaemonConfig(conf);
|
||||
return IpcFactory.GetProvider().GetClient(daemonConfig.Moniker, daemonConfig);
|
||||
}
|
||||
|
||||
private static async Task<(DropItemForBuildXLFile[], string error)> CreateDropItemsForDirectoryAsync(
|
||||
ConfiguredCommand conf,
|
||||
Daemon daemon,
|
||||
string directoryPath,
|
||||
string directoryId,
|
||||
string dropPath,
|
||||
Regex contentFilter)
|
||||
{
|
||||
Contract.Requires(!string.IsNullOrEmpty(directoryPath));
|
||||
Contract.Requires(!string.IsNullOrEmpty(directoryId));
|
||||
Contract.Requires(dropPath != null);
|
||||
|
||||
if (daemon.ApiClient == null)
|
||||
{
|
||||
return (null, "ApiClient is not initialized");
|
||||
}
|
||||
|
||||
DirectoryArtifact directoryArtifact = BuildXL.Ipc.ExternalApi.DirectoryId.Parse(directoryId);
|
||||
|
||||
var maybeResult = await daemon.ApiClient.GetSealedDirectoryContent(directoryArtifact, directoryPath);
|
||||
if (!maybeResult.Succeeded)
|
||||
{
|
||||
return (null, "could not get the directory content from BuildXL server: " + maybeResult.Failure.Describe());
|
||||
}
|
||||
|
||||
var directoryContent = maybeResult.Result;
|
||||
daemon.Logger.Verbose($"(dirPath'{directoryPath}', dirId='{directoryId}') contains '{directoryContent.Count}' files:{Environment.NewLine}{string.Join(Environment.NewLine, directoryContent.Select(f => f.Render()))}");
|
||||
|
||||
if (contentFilter != null)
|
||||
{
|
||||
var filteredContent = directoryContent.Where(file => contentFilter.IsMatch(file.FileName)).ToList();
|
||||
daemon.Logger.Verbose("[dirId='{0}'] Filter '{1}' excluded {2} file(s) out of {3}", directoryId, contentFilter, directoryContent.Count - filteredContent.Count, directoryContent.Count);
|
||||
directoryContent = filteredContent;
|
||||
}
|
||||
|
||||
return (directoryContent.Select(file =>
|
||||
{
|
||||
var remoteFileName = Inv(
|
||||
"{0}/{1}",
|
||||
dropPath,
|
||||
// we need to convert '\' into '/' because this path would be a part of a drop url
|
||||
GetRelativePath(directoryPath, file.FileName).Replace('\\', '/'));
|
||||
|
||||
return new DropItemForBuildXLFile(
|
||||
daemon.ApiClient,
|
||||
file.FileName,
|
||||
BuildXL.Ipc.ExternalApi.FileId.ToString(file.Artifact),
|
||||
conf.Get(EnableChunkDedup),
|
||||
file.ContentInfo,
|
||||
remoteFileName);
|
||||
}).ToArray(), null);
|
||||
}
|
||||
|
||||
private static async Task<(IEnumerable<DropItemForBuildXLFile>, string error)> CreateDropItemsForDirectoriesAsync(
|
||||
ConfiguredCommand conf,
|
||||
Daemon daemon,
|
||||
string[] directoryPaths,
|
||||
string[] directoryIds,
|
||||
string[] dropPaths,
|
||||
Regex[] contentFilters)
|
||||
{
|
||||
Contract.Requires(directoryPaths != null);
|
||||
Contract.Requires(directoryIds != null);
|
||||
Contract.Requires(dropPaths != null);
|
||||
Contract.Requires(contentFilters != null);
|
||||
Contract.Requires(directoryPaths.Length == directoryIds.Length);
|
||||
Contract.Requires(directoryPaths.Length == dropPaths.Length);
|
||||
Contract.Requires(directoryPaths.Length == contentFilters.Length);
|
||||
|
||||
var createDropItemsTasks = Enumerable
|
||||
.Range(0, directoryPaths.Length)
|
||||
.Select(i => CreateDropItemsForDirectoryAsync(conf, daemon, directoryPaths[i], directoryIds[i], dropPaths[i], contentFilters[i])).ToArray();
|
||||
|
||||
var createDropItemsResults = await TaskUtilities.SafeWhenAll(createDropItemsTasks);
|
||||
|
||||
if (createDropItemsResults.Any(r => r.error != null))
|
||||
{
|
||||
return (null, string.Join("; ", createDropItemsResults.Where(r => r.error != null).Select(r => r.error)));
|
||||
}
|
||||
|
||||
return (createDropItemsResults.SelectMany(r => r.Item1), null);
|
||||
}
|
||||
|
||||
private static string GetRelativePath(string root, string file)
|
||||
{
|
||||
var rootEndsWithSlash =
|
||||
root[root.Length - 1] == System.IO.Path.DirectorySeparatorChar
|
||||
|| root[root.Length - 1] == System.IO.Path.AltDirectorySeparatorChar;
|
||||
return file.Substring(root.Length + (rootEndsWithSlash ? 0 : 1));
|
||||
}
|
||||
|
||||
private static (IEnumerable<DropItemForBuildXLFile>, string error) DedupeDropItems(IEnumerable<DropItemForBuildXLFile> dropItems)
|
||||
{
|
||||
var dropItemsByDropPaths = new Dictionary<string, DropItemForBuildXLFile>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var dropItem in dropItems)
|
||||
{
|
||||
if (dropItemsByDropPaths.TryGetValue(dropItem.RelativeDropPath, out var existingDropItem))
|
||||
{
|
||||
if (!string.Equals(dropItem.FullFilePath, existingDropItem.FullFilePath, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return (
|
||||
null,
|
||||
Inv(
|
||||
"'{0}' cannot be added to drop because it has the same drop path '{1}' as '{2}'",
|
||||
dropItem.FullFilePath,
|
||||
dropItem.RelativeDropPath,
|
||||
existingDropItem.FullFilePath));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
dropItemsByDropPaths.Add(dropItem.RelativeDropPath, dropItem);
|
||||
}
|
||||
}
|
||||
|
||||
return (dropItemsByDropPaths.Select(kvp => kvp.Value).ToArray(), null);
|
||||
}
|
||||
|
||||
private static async Task<IIpcResult> AddDropItemsAsync(Daemon daemon, IEnumerable<DropItemForBuildXLFile> dropItems)
|
||||
{
|
||||
(IEnumerable<DropItemForBuildXLFile> dedupedDropItems, string error) = DedupeDropItems(dropItems);
|
||||
|
||||
if (error != null)
|
||||
{
|
||||
return new IpcResult(IpcResultStatus.ExecutionError, error);
|
||||
}
|
||||
|
||||
var ipcResultTasks = dedupedDropItems.Select(d => daemon.AddFileAsync(d)).ToArray();
|
||||
var ipcResults = await TaskUtilities.SafeWhenAll(ipcResultTasks);
|
||||
|
||||
return IpcResult.Merge(ipcResults);
|
||||
}
|
||||
|
||||
private static string Usage()
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
var len = Commands.Keys.Max(cmdName => cmdName.Length);
|
||||
foreach (var cmd in Commands.Values)
|
||||
{
|
||||
builder.AppendLine(Inv(" {0,-" + len + "} : {1}", cmd.Name, cmd.Description));
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
private static int SyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, true);
|
||||
|
||||
private static int AsyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, false);
|
||||
|
||||
private static int RPCSend(ConfiguredCommand conf, IClient rpc, bool isSync)
|
||||
{
|
||||
var rpcResult = RPCSendCore(conf, rpc, isSync);
|
||||
conf.Logger.Info(
|
||||
"Command '{0}' {1} (exit code: {2}). {3}",
|
||||
conf.Command.Name,
|
||||
rpcResult.Succeeded ? "succeeded" : "failed",
|
||||
(int)rpcResult.ExitCode,
|
||||
rpcResult.Payload);
|
||||
return (int)rpcResult.ExitCode;
|
||||
}
|
||||
|
||||
private static IIpcResult RPCSendCore(ConfiguredCommand conf, IClient rpc, bool isSync)
|
||||
{
|
||||
string operationPayload = ToPayload(conf);
|
||||
var operation = new IpcOperation(operationPayload, waitForServerAck: isSync);
|
||||
return rpc.Send(operation).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconstructs a full command line from a command name (<paramref name="commandName"/>)
|
||||
/// and a configuration (<paramref name="config"/>).
|
||||
/// </summary>
|
||||
internal static string ToPayload(string commandName, Config config)
|
||||
{
|
||||
return commandName + " " + config.Render();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconstructs a full command line corresponding to a <see cref="ConfiguredCommand"/>.
|
||||
/// </summary>
|
||||
private static string ToPayload(ConfiguredCommand cmd) => ToPayload(cmd.Command.Name, cmd.Config);
|
||||
|
||||
private static T RegisterOption<T>(List<Option> options, T option) where T : Option
|
||||
{
|
||||
options.Add(option);
|
||||
return option;
|
||||
}
|
||||
|
||||
private static T RegisterDaemonConfigOption<T>(T option) where T : Option => RegisterOption(DaemonConfigOptions, option);
|
||||
|
||||
private static T RegisterDropConfigOption<T>(T option) where T : Option => RegisterOption(DropConfigOptions, option);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,6 +29,7 @@ export namespace DropDaemon {
|
|||
importFrom("BuildXL.Utilities").Ipc.dll,
|
||||
importFrom("BuildXL.Utilities").Native.dll,
|
||||
importFrom("BuildXL.Utilities").Storage.dll,
|
||||
importFrom("BuildXL.Tools").ServicePipDaemon.dll,
|
||||
|
||||
importFrom("ArtifactServices.App.Shared").pkg,
|
||||
importFrom("ArtifactServices.App.Shared.Cache").pkg,
|
||||
|
|
|
@ -26,6 +26,7 @@ using Microsoft.VisualStudio.Services.Drop.WebApi;
|
|||
using Microsoft.VisualStudio.Services.ItemStore.Common;
|
||||
using Newtonsoft.Json;
|
||||
using static BuildXL.Utilities.FormattableStringEx;
|
||||
using Tool.ServicePipDaemon;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
{
|
||||
|
@ -362,7 +363,7 @@ namespace Tool.DropDaemon
|
|||
var notFoundMissingHashes = associateStatus.Missing.Where(b => !providedHashes.Contains(b)).ToList();
|
||||
if (notFoundMissingHashes.Any())
|
||||
{
|
||||
throw new DropDaemonException("This many hashes not found in blobs to upload: " + notFoundMissingHashes.Count());
|
||||
throw new DaemonException("This many hashes not found in blobs to upload: " + notFoundMissingHashes.Count());
|
||||
}
|
||||
#endif
|
||||
|
||||
|
@ -482,7 +483,7 @@ namespace Tool.DropDaemon
|
|||
if (m_fileBlobDescriptorForAssociate != null &&
|
||||
!m_fileBlobDescriptorForAssociate.BlobIdentifier.Equals(m_fileBlobDescriptorForUpload.BlobIdentifier))
|
||||
{
|
||||
throw new DropDaemonException(I($"Blob identifier for file '{m_fileBlobDescriptorForUpload.AbsolutePath}' returned for 'UploadAndAssociate' ({m_fileBlobDescriptorForUpload.BlobIdentifier}) is different from the one returned for 'Associate' ({m_fileBlobDescriptorForAssociate.BlobIdentifier})."));
|
||||
throw new DaemonException(I($"Blob identifier for file '{m_fileBlobDescriptorForUpload.AbsolutePath}' returned for 'UploadAndAssociate' ({m_fileBlobDescriptorForUpload.BlobIdentifier}) is different from the one returned for 'Associate' ({m_fileBlobDescriptorForAssociate.BlobIdentifier})."));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ using BuildXL.Tracing.CloudBuild;
|
|||
using BuildXL.Utilities.Instrumentation.Common;
|
||||
using System.Diagnostics.Tracing;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Implementation of <see cref="ICloudBuildLogger"/> that uses BuildXL's LoggingContext/>.
|
|
@ -0,0 +1,149 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Utilities.CLI;
|
||||
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <nodoc/>
|
||||
public delegate int ClientAction(ConfiguredCommand conf, IClient rpc);
|
||||
|
||||
/// <nodoc/>
|
||||
public delegate Task<IIpcResult> ServerAction(ConfiguredCommand conf, ServicePipDaemon daemon);
|
||||
|
||||
/// <summary>
|
||||
/// A command has a name, description, a list of options it supports, and two actions:
|
||||
/// one for executing this command on the client, and one for executing it on the server.
|
||||
///
|
||||
/// When an instance of a daemon (e.g., DropDaemon) is invoked, command line arguments
|
||||
/// are parsed to determine the command specified by the user. That command is then
|
||||
/// interpreted by executing its <see cref="ClientAction"/>.
|
||||
///
|
||||
/// Most of the daemon commands will be RPC calls, i.e., when a command is received via
|
||||
/// the command line, it is to be marshaled and sent over to a running daemon server via an RPC.
|
||||
/// In such a case, the client action simply invokes <see cref="IClient.Send"/>.
|
||||
///
|
||||
/// When an RPC is received by a daemon server (<see cref="ServicePipDaemon.ParseAndExecuteCommand"/>),
|
||||
/// a <see cref="Command"/> is unmarshaled from the payload of the RPC operation and
|
||||
/// is interpreted on the server by executing its <see cref="ServerAction"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Immutable.
|
||||
/// </remarks>
|
||||
public sealed class Command
|
||||
{
|
||||
/// <summary>A unique command name.</summary>
|
||||
public string Name { get; }
|
||||
|
||||
/// <summary>Arbitrary description.</summary>
|
||||
public string Description { get; }
|
||||
|
||||
/// <summary>Options that may/must be passed to this command.</summary>
|
||||
public IReadOnlyCollection<Option> Options { get; }
|
||||
|
||||
/// <summary>Action to be executed when this command is received via the command line.</summary>
|
||||
public ClientAction ClientAction { get; }
|
||||
|
||||
/// <summary>Action to be executed when this command is received via an RPC call.</summary>
|
||||
public ServerAction ServerAction { get; }
|
||||
|
||||
/// <summary>Whether this command requires an IpcClient; defaults to true.</summary>
|
||||
public bool NeedsIpcClient { get; }
|
||||
|
||||
/// <nodoc />
|
||||
public Command(
|
||||
string name,
|
||||
IEnumerable<Option> options = null,
|
||||
ServerAction serverAction = null,
|
||||
ClientAction clientAction = null,
|
||||
string description = null,
|
||||
bool needsIpcClient = true)
|
||||
{
|
||||
Contract.Requires(name != null);
|
||||
|
||||
Name = name;
|
||||
ServerAction = serverAction;
|
||||
ClientAction = clientAction;
|
||||
Description = description;
|
||||
Options = options.ToList();
|
||||
NeedsIpcClient = needsIpcClient;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs a functional composition of a number of <see cref="ServerAction"/> functions,
|
||||
/// where the results are merged by calling <see cref="IpcResult.Merge(IIpcResult, IIpcResult)"/>.
|
||||
/// </summary>
|
||||
public static ServerAction Compose(params ServerAction[] actions)
|
||||
{
|
||||
Contract.Requires(actions != null);
|
||||
Contract.Requires(actions.Length > 0);
|
||||
|
||||
var first = actions.First();
|
||||
return actions.Skip(1).Aggregate(first, (accumulator, currentAction) => new ServerAction(async (conf, daemon) =>
|
||||
{
|
||||
var lhsResult = await accumulator(conf, daemon);
|
||||
var rhsResult = await currentAction(conf, daemon);
|
||||
return IpcResult.Merge(lhsResult, rhsResult);
|
||||
}));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the usage information for this command.
|
||||
/// </summary>
|
||||
public string Usage(IParser parser)
|
||||
{
|
||||
var result = new StringBuilder();
|
||||
var tab = " ";
|
||||
result.AppendLine("NAME");
|
||||
result.Append(tab).Append(Name).Append(" - ").AppendLine(Description);
|
||||
result.AppendLine();
|
||||
result.AppendLine("SWITCHES");
|
||||
var optsSorted = Options
|
||||
.OrderBy(o => o.IsRequired ? 0 : 1)
|
||||
.ThenBy(o => o.LongName)
|
||||
.ToArray();
|
||||
result.AppendLine(parser.Usage(optsSorted, tab, tab));
|
||||
return result.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Simple wrapper class that holds a <see cref="Command"/> and a <see cref="Config"/>
|
||||
/// containing actual values for the command's <see cref="Command.Options"/>.
|
||||
/// </summary>
|
||||
public sealed class ConfiguredCommand
|
||||
{
|
||||
/// <summary>
|
||||
/// The command.
|
||||
/// </summary>
|
||||
public Command Command { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Configured values for command's options
|
||||
/// </summary>
|
||||
public Config Config { get; }
|
||||
|
||||
/// <nodoc/>
|
||||
public ILogger Logger { get; }
|
||||
|
||||
/// <nodoc/>
|
||||
public ConfiguredCommand(Command command, Config config, ILogger logger)
|
||||
{
|
||||
Command = command;
|
||||
Config = config;
|
||||
Logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the configured value of a particular command option.
|
||||
/// </summary>
|
||||
public T Get<T>(Option<T> option) => option.GetValue(Config);
|
||||
}
|
||||
}
|
|
@ -5,18 +5,20 @@ using System;
|
|||
using System.Diagnostics.ContractsLight;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Daemon configuration.
|
||||
/// </summary>
|
||||
public sealed class DaemonConfig : IServerConfig, IClientConfig
|
||||
{
|
||||
// <nodoc />
|
||||
internal ILogger Logger { get; }
|
||||
/// <nodoc/>
|
||||
public ILogger Logger { get; }
|
||||
|
||||
/// <inheritdoc/>
|
||||
ILogger IServerConfig.Logger => Logger;
|
||||
|
||||
/// <inheritdoc/>
|
||||
ILogger IClientConfig.Logger => Logger;
|
||||
|
||||
#region ConfigOptions
|
|
@ -2,16 +2,17 @@
|
|||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Runtime.Remoting.Messaging;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Generic DropDaemon exception.
|
||||
/// </summary>
|
||||
public sealed class DropDaemonException : Exception
|
||||
public sealed class DaemonException : Exception
|
||||
{
|
||||
/// <nodoc/>
|
||||
public DropDaemonException(string message)
|
||||
public DaemonException(string message)
|
||||
: base(message) { }
|
||||
}
|
||||
}
|
|
@ -3,7 +3,7 @@
|
|||
|
||||
using BuildXL.Tracing.CloudBuild;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Implementation of <see cref="ICloudBuildLogger"/> that doesn't do anything.
|
|
@ -7,7 +7,7 @@ using System.Diagnostics.Tracing;
|
|||
using BuildXL.Tracing;
|
||||
using BuildXL.Tracing.CloudBuild;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Simple listener for Drop events.
|
|
@ -4,7 +4,7 @@
|
|||
using System.Diagnostics.ContractsLight;
|
||||
using BuildXL.Tracing.CloudBuild;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Interface for sending ETW events.
|
|
@ -7,7 +7,7 @@ using System.Diagnostics.ContractsLight;
|
|||
using System.Linq;
|
||||
using JetBrains.Annotations;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Utility class for synchronizing request to "reload" (reconstruct) some underlying value.
|
|
@ -15,10 +15,10 @@ using Microsoft.VisualStudio.Services.Drop.App.Core;
|
|||
using Microsoft.VisualStudio.Services.Drop.WebApi;
|
||||
using Microsoft.VisualStudio.Services.ItemStore.Common;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="IDropServiceClient"/> which will retrie every operation in case <see cref="Microsoft.VisualStudio.Services.Common.VssUnauthorizedException"/> is caught.
|
||||
/// <see cref="IDropServiceClient"/> which will retry every operation in case <see cref="Microsoft.VisualStudio.Services.Common.VssUnauthorizedException"/> is caught.
|
||||
/// </summary>
|
||||
public sealed class ReloadingDropServiceClient : IDropServiceClient
|
||||
{
|
|
@ -0,0 +1,502 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using BuildXL.Ipc;
|
||||
using BuildXL.Ipc.Common;
|
||||
using BuildXL.Ipc.ExternalApi;
|
||||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Utilities;
|
||||
using BuildXL.Utilities.CLI;
|
||||
using BuildXL.Utilities.Tracing;
|
||||
using JetBrains.Annotations;
|
||||
using static BuildXL.Utilities.FormattableStringEx;
|
||||
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Responsible for accepting and handling TCP/IP connections from clients.
|
||||
/// </summary>
|
||||
public abstract class ServicePipDaemon : IDisposable, IIpcOperationExecutor
|
||||
{
|
||||
internal static readonly IIpcProvider IpcProvider = IpcFactory.GetProvider();
|
||||
internal static readonly List<Option> DaemonConfigOptions = new List<Option>();
|
||||
|
||||
/// <summary>Initialized commands</summary>
|
||||
protected static readonly Dictionary<string, Command> Commands = new Dictionary<string, Command>();
|
||||
|
||||
private const string LogFileName = "ServiceDaemon";
|
||||
|
||||
/// <nodoc/>
|
||||
public const string LogPrefix = "(SPD) ";
|
||||
|
||||
/// <summary>Daemon configuration.</summary>
|
||||
public DaemonConfig Config { get; }
|
||||
|
||||
/// <summary>Task to wait on for the completion result.</summary>
|
||||
public Task Completion => m_server.Completion;
|
||||
|
||||
/// <summary>Client for talking to BuildXL.</summary>
|
||||
[CanBeNull]
|
||||
public Client ApiClient { get; }
|
||||
|
||||
/// <nodoc />
|
||||
protected readonly ICloudBuildLogger m_etwLogger;
|
||||
|
||||
/// <nodoc />
|
||||
protected readonly IServer m_server;
|
||||
|
||||
/// <nodoc />
|
||||
protected readonly IParser m_parser;
|
||||
|
||||
/// <nodoc />
|
||||
protected readonly ILogger m_logger;
|
||||
|
||||
/// <nodoc />
|
||||
protected readonly CounterCollection<DaemonCounter> m_counters = new CounterCollection<DaemonCounter>();
|
||||
|
||||
/// <nodoc />
|
||||
protected enum DaemonCounter
|
||||
{
|
||||
/// <nodoc/>
|
||||
[CounterType(CounterType.Stopwatch)]
|
||||
ParseArgsDuration,
|
||||
|
||||
/// <nodoc/>
|
||||
[CounterType(CounterType.Stopwatch)]
|
||||
ServerActionDuration,
|
||||
|
||||
/// <nodoc/>
|
||||
QueueDurationMs,
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
public ILogger Logger => m_logger;
|
||||
|
||||
#region Options and commands
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption Moniker = RegisterDaemonConfigOption(new StrOption("moniker")
|
||||
{
|
||||
ShortName = "m",
|
||||
HelpText = "Moniker to identify client/server communication",
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly IntOption MaxConcurrentClients = RegisterDaemonConfigOption(new IntOption("maxConcurrentClients")
|
||||
{
|
||||
HelpText = "OBSOLETE due to the hardcoded config. (Maximum number of clients to serve concurrently)",
|
||||
DefaultValue = DaemonConfig.DefaultMaxConcurrentClients,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly IntOption MaxConnectRetries = RegisterDaemonConfigOption(new IntOption("maxConnectRetries")
|
||||
{
|
||||
HelpText = "Maximum number of retries to establish a connection with a running daemon",
|
||||
DefaultValue = DaemonConfig.DefaultMaxConnectRetries,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly IntOption ConnectRetryDelayMillis = RegisterDaemonConfigOption(new IntOption("connectRetryDelayMillis")
|
||||
{
|
||||
HelpText = "Delay between consecutive retries to establish a connection with a running daemon",
|
||||
DefaultValue = (int)DaemonConfig.DefaultConnectRetryDelay.TotalMilliseconds,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly BoolOption ShellExecute = RegisterDaemonConfigOption(new BoolOption("shellExecute")
|
||||
{
|
||||
HelpText = "Use shell execute to start the daemon process (a shell window will be created and displayed)",
|
||||
DefaultValue = false,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly BoolOption StopOnFirstFailure = RegisterDaemonConfigOption(new BoolOption("stopOnFirstFailure")
|
||||
{
|
||||
HelpText = "Daemon process should terminate after first failed operation (e.g., 'drop create' fails because the drop already exists).",
|
||||
DefaultValue = DaemonConfig.DefaultStopOnFirstFailure,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly BoolOption EnableCloudBuildIntegration = RegisterDaemonConfigOption(new BoolOption("enableCloudBuildIntegration")
|
||||
{
|
||||
ShortName = "ecb",
|
||||
HelpText = "Enable logging ETW events for CloudBuild to pick up",
|
||||
DefaultValue = DaemonConfig.DefaultEnableCloudBuildIntegration,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly BoolOption Verbose = RegisterDaemonConfigOption(new BoolOption("verbose")
|
||||
{
|
||||
ShortName = "v",
|
||||
HelpText = "Verbose logging",
|
||||
IsRequired = false,
|
||||
DefaultValue = false,
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption LogDir = RegisterDaemonConfigOption(new StrOption("logDir")
|
||||
{
|
||||
ShortName = "log",
|
||||
HelpText = "Log directory",
|
||||
IsRequired = false
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption File = new StrOption("file")
|
||||
{
|
||||
ShortName = "f",
|
||||
HelpText = "File path",
|
||||
IsRequired = false,
|
||||
IsMultiValue = true,
|
||||
};
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption IpcServerMonikerRequired = new StrOption("ipcServerMoniker")
|
||||
{
|
||||
ShortName = "dm",
|
||||
HelpText = "IPC moniker identifying a running BuildXL IPC server",
|
||||
IsRequired = true,
|
||||
};
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption HelpNoNameOption = new StrOption(string.Empty)
|
||||
{
|
||||
HelpText = "Command name",
|
||||
};
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly StrOption IpcServerMonikerOptional = new StrOption(longName: IpcServerMonikerRequired.LongName)
|
||||
{
|
||||
ShortName = IpcServerMonikerRequired.ShortName,
|
||||
HelpText = IpcServerMonikerRequired.HelpText,
|
||||
IsRequired = false,
|
||||
};
|
||||
|
||||
/// <nodoc />
|
||||
protected static T RegisterOption<T>(List<Option> options, T option) where T : Option
|
||||
{
|
||||
options.Add(option);
|
||||
return option;
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
protected static T RegisterDaemonConfigOption<T>(T option) where T : Option => RegisterOption(DaemonConfigOptions, option);
|
||||
|
||||
/// <remarks>
|
||||
/// The <see cref="DaemonConfigOptions"/> options are added to every command.
|
||||
/// A non-mandatory string option "name" is added as well, which operation
|
||||
/// commands may want to use to explicitly specify a particular end point name
|
||||
/// (e.g., the target drop name).
|
||||
/// </remarks>
|
||||
public static Command RegisterCommand(
|
||||
string name,
|
||||
IEnumerable<Option> options = null,
|
||||
ServerAction serverAction = null,
|
||||
ClientAction clientAction = null,
|
||||
string description = null,
|
||||
bool needsIpcClient = true,
|
||||
bool addDaemonConfigOptions = true)
|
||||
{
|
||||
var opts = (options ?? new Option[0]).ToList();
|
||||
if (addDaemonConfigOptions)
|
||||
{
|
||||
opts.AddRange(DaemonConfigOptions);
|
||||
}
|
||||
|
||||
if (!opts.Exists(opt => opt.LongName == "name"))
|
||||
{
|
||||
opts.Add(new Option(longName: "name")
|
||||
{
|
||||
ShortName = "n",
|
||||
});
|
||||
}
|
||||
|
||||
var cmd = new Command(name, opts, serverAction, clientAction, description, needsIpcClient);
|
||||
Commands[cmd.Name] = cmd;
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
protected static readonly Command HelpCmd = RegisterCommand(
|
||||
name: "help",
|
||||
description: "Prints a help message (usage).",
|
||||
options: new[] { HelpNoNameOption },
|
||||
needsIpcClient: false,
|
||||
clientAction: (conf, rpc) =>
|
||||
{
|
||||
string cmdName = conf.Get(HelpNoNameOption);
|
||||
bool cmdNotSpecified = string.IsNullOrWhiteSpace(cmdName);
|
||||
if (cmdNotSpecified)
|
||||
{
|
||||
Console.WriteLine(Usage());
|
||||
return 0;
|
||||
}
|
||||
|
||||
Command requestedHelpForCommand;
|
||||
var requestedCommandFound = Commands.TryGetValue(cmdName, out requestedHelpForCommand);
|
||||
if (requestedCommandFound)
|
||||
{
|
||||
Console.WriteLine(requestedHelpForCommand.Usage(conf.Config.Parser));
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine(Usage());
|
||||
return 1;
|
||||
}
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly Command StopDaemonCmd = RegisterCommand(
|
||||
name: "stop",
|
||||
description: "[RPC] Stops the daemon process running on specified port; fails if no such daemon is running.",
|
||||
clientAction: AsyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
conf.Logger.Info("[STOP] requested");
|
||||
daemon.RequestStop();
|
||||
return Task.FromResult(IpcResult.Success());
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly Command CrashDaemonCmd = RegisterCommand(
|
||||
name: "crash",
|
||||
description: "[RPC] Stops the server process by crashing it.",
|
||||
clientAction: AsyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[CRASH] requested");
|
||||
Environment.Exit(-1);
|
||||
return Task.FromResult(IpcResult.Success());
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly Command PingDaemonCmd = RegisterCommand(
|
||||
name: "ping",
|
||||
description: "[RPC] Pings the daemon process.",
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[PING] received");
|
||||
return Task.FromResult(IpcResult.Success("Alive!"));
|
||||
});
|
||||
|
||||
/// <nodoc />
|
||||
public static readonly Command TestReadFile = RegisterCommand(
|
||||
name: "test-readfile",
|
||||
description: "[RPC] Sends a request to the daemon to read a file.",
|
||||
options: new Option[] { File },
|
||||
clientAction: SyncRPCSend,
|
||||
serverAction: (conf, daemon) =>
|
||||
{
|
||||
daemon.Logger.Info("[READFILE] received");
|
||||
var result = IpcResult.Success(System.IO.File.ReadAllText(conf.Get(File)));
|
||||
daemon.Logger.Info("[READFILE] succeeded");
|
||||
return Task.FromResult(result);
|
||||
});
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
/// <nodoc />
|
||||
public ServicePipDaemon(IParser parser, DaemonConfig daemonConfig, ILogger logger, IIpcProvider rpcProvider = null, Client client = null)
|
||||
{
|
||||
Contract.Requires(daemonConfig != null);
|
||||
|
||||
Config = daemonConfig;
|
||||
m_parser = parser;
|
||||
ApiClient = client;
|
||||
m_logger = logger;
|
||||
|
||||
rpcProvider = rpcProvider ?? IpcFactory.GetProvider();
|
||||
m_server = rpcProvider.GetServer(Config.Moniker, Config);
|
||||
|
||||
m_etwLogger = new BuildXLBasedCloudBuildLogger(Config.Logger, Config.EnableCloudBuildIntegration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts to listen for client connections. As soon as a connection is received,
|
||||
/// it is placed in an action block from which it is picked up and handled asynchronously
|
||||
/// (in the <see cref="ParseAndExecuteCommand"/> method).
|
||||
/// </summary>
|
||||
public void Start()
|
||||
{
|
||||
m_server.Start(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Requests shut down, causing this daemon to immediately stop listening for TCP/IP
|
||||
/// connections. Any pending requests, however, will be processed to completion.
|
||||
/// </summary>
|
||||
public void RequestStop()
|
||||
{
|
||||
m_server.RequestStop();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calls <see cref="RequestStop"/> then waits for <see cref="Completion"/>.
|
||||
/// </summary>
|
||||
public Task RequestStopAndWaitForCompletionAsync()
|
||||
{
|
||||
RequestStop();
|
||||
return Completion;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
m_server.Dispose();
|
||||
ApiClient?.Dispose();
|
||||
m_logger.Dispose();
|
||||
}
|
||||
|
||||
private async Task<IIpcResult> ParseAndExecuteCommand(IIpcOperation operation)
|
||||
{
|
||||
string cmdLine = operation.Payload;
|
||||
m_logger.Verbose("Command received: {0}", cmdLine);
|
||||
ConfiguredCommand conf;
|
||||
using (m_counters.StartStopwatch(DaemonCounter.ParseArgsDuration))
|
||||
{
|
||||
conf = ParseArgs(cmdLine, m_parser);
|
||||
}
|
||||
|
||||
IIpcResult result;
|
||||
using (var duration = m_counters.StartStopwatch(DaemonCounter.ServerActionDuration))
|
||||
{
|
||||
result = await conf.Command.ServerAction(conf, this);
|
||||
result.ActionDuration = duration.Elapsed;
|
||||
}
|
||||
|
||||
TimeSpan queueDuration = operation.Timestamp.Daemon_BeforeExecuteTime - operation.Timestamp.Daemon_AfterReceivedTime;
|
||||
m_counters.AddToCounter(DaemonCounter.QueueDurationMs, (long)queueDuration.TotalMilliseconds);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
Task<IIpcResult> IIpcOperationExecutor.ExecuteAsync(IIpcOperation operation)
|
||||
{
|
||||
Contract.Requires(operation != null);
|
||||
|
||||
return ParseAndExecuteCommand(operation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a string and returns a ConfiguredCommand.
|
||||
/// </summary>
|
||||
public static ConfiguredCommand ParseArgs(string allArgs, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
|
||||
{
|
||||
return ParseArgs(parser.SplitArgs(allArgs), parser, logger, ignoreInvalidOptions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a list of arguments and returns a ConfiguredCommand.
|
||||
/// </summary>
|
||||
public static ConfiguredCommand ParseArgs(string[] args, IParser parser, ILogger logger = null, bool ignoreInvalidOptions = false)
|
||||
{
|
||||
var usageMessage = Lazy.Create(() => "Usage:" + Environment.NewLine + Usage());
|
||||
|
||||
if (args.Length == 0)
|
||||
{
|
||||
throw new ArgumentException(I($"Command is required. {usageMessage.Value}"));
|
||||
}
|
||||
|
||||
var argsQueue = new Queue<string>(args);
|
||||
string cmdName = argsQueue.Dequeue();
|
||||
if (!Commands.TryGetValue(cmdName, out Command cmd))
|
||||
{
|
||||
throw new ArgumentException(I($"No command '{cmdName}' is found. {usageMessage.Value}"));
|
||||
}
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
Config conf = BuildXL.Utilities.CLI.Config.ParseCommandLineArgs(cmd.Options, argsQueue, parser, caseInsensitive: true, ignoreInvalidOptions: ignoreInvalidOptions);
|
||||
var parseTime = sw.Elapsed;
|
||||
|
||||
logger = logger ?? new ConsoleLogger(Verbose.GetValue(conf), ServicePipDaemon.LogPrefix);
|
||||
logger.Verbose("Parsing command line arguments done in {0}", parseTime);
|
||||
return new ConfiguredCommand(cmd, conf, logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates DaemonConfig using the values specified on the ConfiguredCommand
|
||||
/// </summary>
|
||||
public static DaemonConfig CreateDaemonConfig(ConfiguredCommand conf)
|
||||
{
|
||||
return new DaemonConfig(
|
||||
logger: conf.Logger,
|
||||
moniker: conf.Get(Moniker),
|
||||
maxConnectRetries: conf.Get(MaxConnectRetries),
|
||||
connectRetryDelay: TimeSpan.FromMilliseconds(conf.Get(ConnectRetryDelayMillis)),
|
||||
stopOnFirstFailure: conf.Get(StopOnFirstFailure),
|
||||
enableCloudBuildIntegration: conf.Get(EnableCloudBuildIntegration));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates anIPC client using the config from a ConfiguredCommand
|
||||
/// </summary>
|
||||
public static IClient CreateClient(ConfiguredCommand conf)
|
||||
{
|
||||
var daemonConfig = CreateDaemonConfig(conf);
|
||||
return IpcProvider.GetClient(daemonConfig.Moniker, daemonConfig);
|
||||
}
|
||||
|
||||
private static string Usage()
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
var len = Commands.Keys.Max(cmdName => cmdName.Length);
|
||||
foreach (var cmd in Commands.Values)
|
||||
{
|
||||
builder.AppendLine(string.Format(CultureInfo.InvariantCulture, " {0,-" + len + "} : {1}", cmd.Name, cmd.Description));
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
protected static int SyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, true);
|
||||
|
||||
/// <nodoc />
|
||||
protected static int AsyncRPCSend(ConfiguredCommand conf, IClient rpc) => RPCSend(conf, rpc, false);
|
||||
|
||||
/// <nodoc />
|
||||
protected static int RPCSend(ConfiguredCommand conf, IClient rpc, bool isSync)
|
||||
{
|
||||
var rpcResult = RPCSendCore(conf, rpc, isSync);
|
||||
conf.Logger.Info(
|
||||
"Command '{0}' {1} (exit code: {2}). {3}",
|
||||
conf.Command.Name,
|
||||
rpcResult.Succeeded ? "succeeded" : "failed",
|
||||
(int)rpcResult.ExitCode,
|
||||
rpcResult.Payload);
|
||||
return (int)rpcResult.ExitCode;
|
||||
}
|
||||
|
||||
private static IIpcResult RPCSendCore(ConfiguredCommand conf, IClient rpc, bool isSync)
|
||||
{
|
||||
string operationPayload = ToPayload(conf);
|
||||
var operation = new IpcOperation(operationPayload, waitForServerAck: isSync);
|
||||
return rpc.Send(operation).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconstructs a full command line from a command name (<paramref name="commandName"/>)
|
||||
/// and a configuration (<paramref name="config"/>).
|
||||
/// </summary>
|
||||
internal static string ToPayload(string commandName, Config config)
|
||||
{
|
||||
return commandName + " " + config.Render();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconstructs a full command line corresponding to a <see cref="ConfiguredCommand"/>.
|
||||
/// </summary>
|
||||
private static string ToPayload(ConfiguredCommand cmd) => ToPayload(cmd.Command.Name, cmd.Config);
|
||||
}
|
||||
}
|
|
@ -2,36 +2,24 @@
|
|||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using System.Globalization;
|
||||
using BuildXL.Native.IO;
|
||||
|
||||
namespace Tool.DropDaemon
|
||||
namespace Tool.ServicePipDaemon
|
||||
{
|
||||
/// <summary>
|
||||
/// Various helper method, typically to be imported with "using static".
|
||||
/// </summary>
|
||||
public static class Statics
|
||||
{
|
||||
/// <summary>
|
||||
/// String format with <see cref="CultureInfo.InvariantCulture"/>.
|
||||
/// </summary>
|
||||
public static string Inv(string format, params object[] args)
|
||||
{
|
||||
Contract.Requires(format != null);
|
||||
|
||||
return string.Format(CultureInfo.InvariantCulture, format, args);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logs an error as a line of text. Currently prints out to <code>Console.Error</code>.
|
||||
/// to use whatever other
|
||||
/// </summary>
|
||||
public static void Error(string format, params object[] args)
|
||||
public static void Error(string error)
|
||||
{
|
||||
if (format != null)
|
||||
if (error != null)
|
||||
{
|
||||
Console.Error.WriteLine(Inv(format, args));
|
||||
Console.Error.WriteLine(error);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
import {Artifact, Cmd, Transformer} from "Sdk.Transformers";
|
||||
|
||||
import * as BuildXLSdk from "Sdk.BuildXL";
|
||||
import * as Managed from "Sdk.Managed";
|
||||
import * as Deployment from "Sdk.Deployment";
|
||||
import { NetFx } from "Sdk.BuildXL";
|
||||
|
||||
namespace ServicePipDaemon {
|
||||
|
||||
@@public
|
||||
export const dll = !BuildXLSdk.isDaemonToolingEnabled ? undefined : BuildXLSdk.library({
|
||||
assemblyName: "Tool.ServicePipDaemon",
|
||||
rootNamespace: "Tool.ServicePipDaemon",
|
||||
sources: globR(d`.`, "*.cs"),
|
||||
references:[
|
||||
importFrom("BuildXL.Engine").Scheduler.dll,
|
||||
importFrom("BuildXL.Utilities.Instrumentation").Common.dll,
|
||||
importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll,
|
||||
importFrom("BuildXL.Utilities").dll,
|
||||
importFrom("BuildXL.Utilities").Ipc.dll,
|
||||
importFrom("BuildXL.Utilities").Native.dll,
|
||||
importFrom("BuildXL.Utilities").Storage.dll,
|
||||
|
||||
importFrom("ArtifactServices.App.Shared").pkg,
|
||||
importFrom("ArtifactServices.App.Shared.Cache").pkg,
|
||||
importFrom("Drop.App.Core").pkg,
|
||||
importFrom("Drop.Client").pkg,
|
||||
importFrom("Drop.RemotableClient.Interfaces").pkg,
|
||||
importFrom("ItemStore.Shared").pkg,
|
||||
importFrom("Microsoft.ApplicationInsights").pkg,
|
||||
importFrom("Microsoft.AspNet.WebApi.Client").pkg,
|
||||
importFrom("Microsoft.Diagnostics.Tracing.TraceEvent").pkg,
|
||||
importFrom("Microsoft.IdentityModel.Clients.ActiveDirectory").pkg,
|
||||
...BuildXLSdk.visualStudioServicesArtifactServicesSharedPkg,
|
||||
importFrom("Microsoft.VisualStudio.Services.BlobStore.Client").pkg,
|
||||
importFrom("Microsoft.VisualStudio.Services.Client").pkg,
|
||||
importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
|
||||
importFrom("Newtonsoft.Json").pkg,
|
||||
importFrom("WindowsAzure.Storage").pkg,
|
||||
|
||||
],
|
||||
internalsVisibleTo: [
|
||||
"Test.Tool.DropDaemon",
|
||||
]
|
||||
});
|
||||
}
|
|
@ -11,9 +11,11 @@ using BuildXL.Ipc.Interfaces;
|
|||
using BuildXL.Utilities.CLI;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Tool.DropDaemon;
|
||||
using Tool.ServicePipDaemon;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using static Tool.DropDaemon.Program;
|
||||
using static Tool.DropDaemon.DropDaemon;
|
||||
using static Tool.ServicePipDaemon.ServicePipDaemon;
|
||||
|
||||
namespace Test.Tool.DropDaemon
|
||||
{
|
||||
|
@ -69,18 +71,18 @@ namespace Test.Tool.DropDaemon
|
|||
|
||||
public static IEnumerable<object[]> ValidCommandLinesTestData()
|
||||
{
|
||||
yield return new object[] { "start", new Option[] { DropName, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
|
||||
yield return new object[] { "start", new Option[] { DropName, DropEndpoint, EnableCloudBuildIntegration }, new[] { "mydrop", "http://xyz", "True" } };
|
||||
yield return new object[] { "create", new Option[] { DropName, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
|
||||
yield return new object[] { "addfile", new[] { File, DropName }, new[] { @"""c:\x\y.txt""", "mydrop" } };
|
||||
yield return new object[] { "start", new Option[] { DropNameOption, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
|
||||
yield return new object[] { "start", new Option[] { DropNameOption, DropEndpoint, EnableCloudBuildIntegration }, new[] { "mydrop", "http://xyz", "True" } };
|
||||
yield return new object[] { "create", new Option[] { DropNameOption, DropEndpoint }, new[] { "mydrop", "http://xyz" } };
|
||||
yield return new object[] { "addfile", new[] { File, DropNameOption }, new[] { @"""c:\x\y.txt""", "mydrop" } };
|
||||
yield return new object[] { "addfile", new[] { File }, new[] { @"""c:\x\y.txt""" } };
|
||||
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", "a/b/c.txt" } };
|
||||
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", @"a\\b\\c.txt" } };
|
||||
yield return new object[] { "addfile", new[] { File, RelativeDropPath }, new[] { @"""c:\x\y.txt""", "\"a\\b\\c.txt\"" } };
|
||||
yield return new object[] { "finalize", new Option[0], new string[0] };
|
||||
yield return new object[] { "finalize", new[] { DropName }, new[] { "mydrop" } };
|
||||
yield return new object[] { "finalize", new[] { DropNameOption }, new[] { "mydrop" } };
|
||||
yield return new object[] { "stop", new Option[0], new string[0] };
|
||||
yield return new object[] { "stop", new[] { DropName }, new[] { "mydrop" } };
|
||||
yield return new object[] { "stop", new[] { DropNameOption }, new[] { "mydrop" } };
|
||||
yield return new object[] { "addartifacts", new[] { IpcServerMonikerRequired, Directory, DirectoryId, RelativeDirectoryDropPath }, new[] { "moniker_string", @"c:\dir", "123:1:12345", "/remote/" } };
|
||||
yield return new object[] { "addartifacts", new[] { IpcServerMonikerRequired, File, FileId, HashOptional, RelativeDropPath }, new[] { "moniker_string", @"c:\dir\f.txt", "id1", "123:1:12345", "/remote/f.txt" } };
|
||||
}
|
||||
|
@ -109,11 +111,11 @@ namespace Test.Tool.DropDaemon
|
|||
|
||||
public static IEnumerable<object[]> OptionOverrideTestData()
|
||||
{
|
||||
yield return new object[] { DropName, "a", "b", "c" };
|
||||
yield return new object[] { DropName, "a", "b", null };
|
||||
yield return new object[] { DropName, "a", null, "c" };
|
||||
yield return new object[] { DropName, null, "b", "c" };
|
||||
yield return new object[] { DropName, null, "b", null };
|
||||
yield return new object[] { DropNameOption, "a", "b", "c" };
|
||||
yield return new object[] { DropNameOption, "a", "b", null };
|
||||
yield return new object[] { DropNameOption, "a", null, "c" };
|
||||
yield return new object[] { DropNameOption, null, "b", "c" };
|
||||
yield return new object[] { DropNameOption, null, "b", null };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
@ -133,7 +135,8 @@ namespace Test.Tool.DropDaemon
|
|||
|
||||
public static IEnumerable<object[]> ParseArgsRequiredOptionMissingTestData()
|
||||
{
|
||||
foreach (var cmd in Commands.Values)
|
||||
// TODO: this only iterates over DropDaemon & ServicePipDaemon commands => add symbols commands
|
||||
foreach (var cmd in SupportedCommands)
|
||||
{
|
||||
var requiredSwitches = cmd.Options.Where(opt => opt.IsRequired).Select(opt => opt.LongName).ToList();
|
||||
if (!requiredSwitches.Any())
|
||||
|
@ -169,8 +172,8 @@ namespace Test.Tool.DropDaemon
|
|||
|
||||
// get either DaemonConfig or DropConfig object from the parsed command line, based on given 'property'
|
||||
object configObject =
|
||||
property.DeclaringType == typeof(DaemonConfig) ? (object)CreateDaemonConfig(cliConfig) :
|
||||
property.DeclaringType == typeof(DropConfig) ? (object)CreateDropConfig(cliConfig) :
|
||||
property.DeclaringType == typeof(DaemonConfig) ? (object)ServicePipDaemon.CreateDaemonConfig(cliConfig) :
|
||||
property.DeclaringType == typeof(DropConfig) ? (object)global::Tool.DropDaemon.DropDaemon.CreateDropConfig(cliConfig) :
|
||||
null;
|
||||
|
||||
// assert the value of that property is what we expect.
|
||||
|
@ -204,7 +207,7 @@ namespace Test.Tool.DropDaemon
|
|||
var cmd = StartCmd;
|
||||
var cmdline = cmd.Name + " --name 123 --service http://xyz "; // prepend required flags in all cases
|
||||
|
||||
foreach (var prop in confType.GetProperties().Where(p => !p.Name.StartsWith("Default") && !p.Name.StartsWith("MaxConcurrentClients")))
|
||||
foreach (var prop in confType.GetProperties().Where(p => !p.Name.StartsWith("Default") && !p.Name.StartsWith("MaxConcurrentClients") && !p.Name.StartsWith("Logger")))
|
||||
{
|
||||
// For current 'prop', try to find a corresponding option ('opt') in 'cmd.Options' and pick a
|
||||
// representative value (according to its type) include in the command line.
|
||||
|
@ -327,7 +330,7 @@ namespace Test.Tool.DropDaemon
|
|||
private ConfiguredCommand ParseArgs(string fullCmdLine, bool ignoreInvalidOptions = false)
|
||||
{
|
||||
var logger = new LambdaLogger((level, format, args) => Output.WriteLine(format, args));
|
||||
return Program.ParseArgs(fullCmdLine, UnixParser.Instance, logger, ignoreInvalidOptions: ignoreInvalidOptions);
|
||||
return global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(fullCmdLine, UnixParser.Instance, logger, ignoreInvalidOptions: ignoreInvalidOptions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -11,9 +11,11 @@ using BuildXL.Ipc.Common.Multiplexing;
|
|||
using BuildXL.Ipc.Interfaces;
|
||||
using BuildXL.Utilities.CLI;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Tool.DropDaemon;
|
||||
using Tool.ServicePipDaemon;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
using static Tool.DropDaemon.DropDaemon;
|
||||
using static Tool.ServicePipDaemon.ServicePipDaemon;
|
||||
|
||||
namespace Test.Tool.DropDaemon
|
||||
{
|
||||
|
@ -42,18 +44,18 @@ namespace Test.Tool.DropDaemon
|
|||
.ToList();
|
||||
|
||||
var serverThreads = ipcMonikers
|
||||
.Select(moniker => CreateThreadForCommand($"{Program.StartNoDropCmd.Name} --{Program.Moniker.LongName} " + moniker, null))
|
||||
.Select(moniker => CreateThreadForCommand($"{StartNoDropCmd.Name} --{Moniker.LongName} " + moniker, null))
|
||||
.ToList();
|
||||
Start(serverThreads);
|
||||
|
||||
Thread.Sleep(100);
|
||||
|
||||
var clientThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, numRequestsPerService, $"{Program.PingDaemonCmd.Name} --{Program.Moniker.LongName} <moniker>");
|
||||
var clientThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, numRequestsPerService, $"{PingDaemonCmd.Name} --{Moniker.LongName} <moniker>");
|
||||
|
||||
Start(clientThreads);
|
||||
Join(clientThreads);
|
||||
|
||||
var serverShutdownThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, 1, $"{Program.StopDaemonCmd.Name} --{Program.Moniker.LongName} <moniker>");
|
||||
var serverShutdownThreads = GetClientThreads(ipcProvider, ipcMonikers, numServices, 1, $"{StopDaemonCmd.Name} --{Moniker.LongName} <moniker>");
|
||||
Start(serverShutdownThreads);
|
||||
Join(serverShutdownThreads);
|
||||
|
||||
|
@ -98,7 +100,7 @@ namespace Test.Tool.DropDaemon
|
|||
Console.WriteLine(format);
|
||||
Output.WriteLine(format);
|
||||
});
|
||||
ConfiguredCommand conf = Program.ParseArgs(cmdLine, UnixParser.Instance, logger);
|
||||
ConfiguredCommand conf = ParseArgs(cmdLine, UnixParser.Instance, logger);
|
||||
var exitCode = conf.Command.ClientAction(conf, client);
|
||||
Assert.Equal(0, exitCode);
|
||||
});
|
||||
|
|
|
@ -21,6 +21,7 @@ using Microsoft.VisualStudio.Services.Common;
|
|||
using Microsoft.VisualStudio.Services.Drop.WebApi;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Tool.DropDaemon;
|
||||
using Tool.ServicePipDaemon;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
|
@ -184,7 +185,7 @@ namespace Test.Tool.DropDaemon
|
|||
XAssert.IsTrue(result.Succeeded);
|
||||
|
||||
// calling MaterializeFile fails because no BuildXL server is running
|
||||
Assert.Throws<DropDaemonException>(() => addFileItem.EnsureMaterialized().GetAwaiter().GetResult());
|
||||
Assert.Throws<DaemonException>(() => addFileItem.EnsureMaterialized().GetAwaiter().GetResult());
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -259,7 +260,7 @@ namespace Test.Tool.DropDaemon
|
|||
var dropClient = new MockDropClient(addFileFunc: (item) =>
|
||||
{
|
||||
Assert.NotNull(item.BlobIdentifier);
|
||||
var ex = Assert.Throws<DropDaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
|
||||
var ex = Assert.Throws<DaemonException>(() => item.EnsureMaterialized().GetAwaiter().GetResult());
|
||||
|
||||
// rethrowing because that's what a real IDropClient would do (then Daemon is expected to handle it)
|
||||
throw ex;
|
||||
|
@ -311,7 +312,7 @@ namespace Test.Tool.DropDaemon
|
|||
{
|
||||
var ipcResult = await daemon.AddFileAsync(new DropItemForFile(targetFile), symlinkTester: (file) => file == targetFile ? true : false);
|
||||
Assert.False(ipcResult.Succeeded, "adding symlink to drop succeeded while it was expected to fail");
|
||||
Assert.True(ipcResult.Payload.Contains(Daemon.SymlinkAddErrorMessagePrefix));
|
||||
Assert.True(ipcResult.Payload.Contains(global::Tool.DropDaemon.DropDaemon.SymlinkAddErrorMessagePrefix));
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -382,7 +383,9 @@ namespace Test.Tool.DropDaemon
|
|||
dropClient,
|
||||
(daemon, etwListener) =>
|
||||
{
|
||||
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}", new UnixParser());
|
||||
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
|
||||
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter {filter}",
|
||||
new UnixParser());
|
||||
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
|
||||
|
||||
XAssert.IsTrue(ipcResult.Succeeded, ipcResult.Payload);
|
||||
|
@ -415,7 +418,9 @@ namespace Test.Tool.DropDaemon
|
|||
{
|
||||
// only hash and file rewrite count are important here; the rest are just fake values
|
||||
var hash = FileContentInfo.CreateWithUnknownLength(ContentHashingUtilities.CreateSpecialValue(1)).Render();
|
||||
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {daemon.Config.Moniker} --file non-existent-file.txt --dropPath remote-file-name.txt --hash {hash} --fileId 12345:{(isSourceFile ? 0 : 1)}", new UnixParser());
|
||||
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
|
||||
$"addartifacts --ipcServerMoniker {daemon.Config.Moniker} --file non-existent-file.txt --dropPath remote-file-name.txt --hash {hash} --fileId 12345:{(isSourceFile ? 0 : 1)}",
|
||||
new UnixParser());
|
||||
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
|
||||
|
||||
XAssert.IsTrue(dropPaths.Count == 0);
|
||||
|
@ -473,7 +478,9 @@ namespace Test.Tool.DropDaemon
|
|||
dropClient,
|
||||
(daemon, etwListener) =>
|
||||
{
|
||||
var addArtifactsCommand = Program.ParseArgs($"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*", new UnixParser());
|
||||
var addArtifactsCommand = global::Tool.ServicePipDaemon.ServicePipDaemon.ParseArgs(
|
||||
$"addartifacts --ipcServerMoniker {moniker.Id} --directory {directoryPath} --directoryId {fakeDirectoryId} --directoryDropPath {remoteDirectoryPath} --directoryFilter .*",
|
||||
new UnixParser());
|
||||
var ipcResult = addArtifactsCommand.Command.ServerAction(addArtifactsCommand, daemon).GetAwaiter().GetResult();
|
||||
|
||||
XAssert.IsTrue(dropPaths.Count == 0);
|
||||
|
@ -532,13 +539,13 @@ namespace Test.Tool.DropDaemon
|
|||
Assert.Equal(shouldSucceed, rpcResult.Succeeded);
|
||||
}
|
||||
|
||||
private void WithSetup(IDropClient dropClient, Action<Daemon, DropEtwListener> action, Client apiClient = null)
|
||||
private void WithSetup(IDropClient dropClient, Action<global::Tool.DropDaemon.DropDaemon, DropEtwListener> action, Client apiClient = null)
|
||||
{
|
||||
var etwListener = ConfigureEtwLogging();
|
||||
string moniker = Program.IpcProvider.RenderConnectionString(Program.IpcProvider.CreateNewMoniker());
|
||||
string moniker = ServicePipDaemon.IpcProvider.RenderConnectionString(ServicePipDaemon.IpcProvider.CreateNewMoniker());
|
||||
var daemonConfig = new DaemonConfig(VoidLogger.Instance, moniker: moniker, enableCloudBuildIntegration: false);
|
||||
var dropConfig = new DropConfig(string.Empty, null);
|
||||
var daemon = new Daemon(UnixParser.Instance, daemonConfig, dropConfig, Task.FromResult(dropClient), client: apiClient);
|
||||
var daemon = new global::Tool.DropDaemon.DropDaemon(UnixParser.Instance, daemonConfig, dropConfig, Task.FromResult(dropClient), client: apiClient);
|
||||
action(daemon, etwListener);
|
||||
}
|
||||
|
||||
|
|
|
@ -5,7 +5,7 @@ using System;
|
|||
using System.Linq;
|
||||
using System.Threading;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Tool.DropDaemon;
|
||||
using Tool.ServicePipDaemon;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
|
|
|
@ -15,7 +15,7 @@ using Microsoft.VisualStudio.Services.Drop.App.Core;
|
|||
using Microsoft.VisualStudio.Services.Drop.WebApi;
|
||||
using Microsoft.VisualStudio.Services.ItemStore.Common;
|
||||
using Test.BuildXL.TestUtilities.Xunit;
|
||||
using Tool.DropDaemon;
|
||||
using Tool.ServicePipDaemon;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ namespace Test.Tool.DropDaemon {
|
|||
importFrom("BuildXL.Cache.ContentStore").UtilitiesCore.dll,
|
||||
importFrom("BuildXL.Engine").Scheduler.dll,
|
||||
importFrom("BuildXL.Tools.DropDaemon").exe,
|
||||
importFrom("BuildXL.Tools").ServicePipDaemon.dll,
|
||||
importFrom("BuildXL.Utilities").dll,
|
||||
importFrom("BuildXL.Utilities").Ipc.dll,
|
||||
importFrom("BuildXL.Utilities").Storage.dll,
|
||||
|
|
|
@ -50,23 +50,32 @@ namespace BuildXL.Ipc.Common
|
|||
/// <nodoc />
|
||||
public IpcResultTimestamp Timestamp { get; }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public TimeSpan ActionDuration { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a result representing a successful IPC operation.
|
||||
/// </summary>
|
||||
public static IIpcResult Success(string payload = null) => new IpcResult(IpcResultStatus.Success, payload);
|
||||
|
||||
/// <nodoc />
|
||||
public IpcResult(IpcResultStatus status, string payload)
|
||||
public IpcResult(IpcResultStatus status, string payload) : this(status, payload, TimeSpan.Zero)
|
||||
{
|
||||
}
|
||||
|
||||
/// <nodoc />
|
||||
public IpcResult(IpcResultStatus status, string payload, TimeSpan actionDuraion)
|
||||
{
|
||||
m_exitCode = status;
|
||||
m_payload = payload ?? string.Empty;
|
||||
Timestamp = new IpcResultTimestamp();
|
||||
ActionDuration = actionDuraion;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override string ToString()
|
||||
{
|
||||
return I($"{{succeeded: {Succeeded}, payload: '{Payload}'}}");
|
||||
return I($"{{succeeded: {Succeeded}, payload: '{Payload}', ActionDuration: {ActionDuration}}}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
@ -90,6 +99,7 @@ namespace BuildXL.Ipc.Common
|
|||
{
|
||||
await Utils.WriteByteAsync(stream, (byte)result.ExitCode, token);
|
||||
await Utils.WriteStringAsync(stream, result.Payload, token);
|
||||
await Utils.WriteLongAsync(stream, result.ActionDuration.Ticks, token);
|
||||
await stream.FlushAsync(token);
|
||||
}
|
||||
|
||||
|
@ -104,7 +114,9 @@ namespace BuildXL.Ipc.Common
|
|||
byte statusByte = await Utils.ReadByteAsync(stream, token);
|
||||
Utils.CheckSerializationFormat(Enum.IsDefined(typeof(IpcResultStatus), statusByte), "unknown IpcResult.Status byte: {0}", statusByte);
|
||||
string payload = await Utils.ReadStringAsync(stream, token);
|
||||
return new IpcResult((IpcResultStatus)statusByte, payload);
|
||||
long actionDuration = await Utils.ReadLongAsync(stream, token);
|
||||
|
||||
return new IpcResult((IpcResultStatus)statusByte, payload, TimeSpan.FromTicks(actionDuration));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
@ -120,7 +132,7 @@ namespace BuildXL.Ipc.Common
|
|||
lhs.Succeeded && !rhs.Succeeded ? rhs.ExitCode :
|
||||
!lhs.Succeeded && rhs.Succeeded ? lhs.ExitCode :
|
||||
IpcResultStatus.GenericError;
|
||||
return new IpcResult(mergedStatus, lhs.Payload + Environment.NewLine + rhs.Payload);
|
||||
return new IpcResult(mergedStatus, lhs.Payload + Environment.NewLine + rhs.Payload, lhs.ActionDuration + rhs.ActionDuration);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Microsoft. All rights reserved.
|
||||
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
|
||||
|
||||
using System;
|
||||
using System.Diagnostics.ContractsLight;
|
||||
using BuildXL.Ipc.Common;
|
||||
|
||||
|
@ -71,5 +72,10 @@ namespace BuildXL.Ipc.Interfaces
|
|||
|
||||
/// <nodoc/>
|
||||
IpcResultTimestamp Timestamp { get; }
|
||||
|
||||
/// <summary>
|
||||
/// (Optional) Duration of the action executed by a server.
|
||||
/// </summary>
|
||||
TimeSpan ActionDuration { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
Загрузка…
Ссылка в новой задаче