microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 675572: Update AdoBuildRunner and include it in the deployment 

Renames the "Orchestrator" project to "AdoBuildRunner" to avoid confusion with the other uses of "Orchestrator" for distributed builds, and includes the tool in the BuildXL deployment. Also some minor tweaks to the tool + a test mode for connectivity tests between agents

Related work items: #1977690
This commit is contained in:
Marcelo Lynch 🧉 2022-08-24 20:14:05 +00:00
Родитель 03efba83ac
Коммит a789aa51e1
19 изменённых файлов: 388 добавлений и 89 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
Public/Src/Deployment/NugetPackages.dsc
Просмотреть файл

@ -338,9 +338,9 @@ namespace NugetPackages {
});
// Currently we deploy tools as self-contained .NET Core binaries for macOS only!
const toolsOrchestrator = pack({
id: `${packageNamePrefix}.Tools.Orchestrator.osx-x64`,
deployment: Tools.Orchestrator.withQualifier({
const toolsAdoBuildRunner = pack({
id: `${packageNamePrefix}.Tools.AdoBuildRunner.osx-x64`,
deployment: Tools.AdoBuildRunner.withQualifier({
targetFramework: defaultTargetFramework,
targetRuntime: "osx-x64"
}).deployment
@ -363,7 +363,7 @@ namespace NugetPackages {
engineCache
]),
sdks,
...addIf(!BuildXLSdk.Flags.genVSSolution, osxX64, linuxX64, toolsOrchestrator),
...addIf(!BuildXLSdk.Flags.genVSSolution, osxX64, linuxX64, toolsAdoBuildRunner),
toolsSandBoxExec
]
};

7
Public/Src/Deployment/buildXL.dsc
Просмотреть файл

@ -35,7 +35,10 @@ namespace BuildXL {
importFrom("BuildXL.Tools").CMakeRunner.exe,
importFrom("BuildXL.Tools").NinjaGraphBuilder.exe,
importFrom("BuildXL.Tools.Ninjson").pkg.contents
])
]),
// ado build runner
importFrom("BuildXL.Tools").AdoBuildRunner.exe,
]
};
@ -46,4 +49,4 @@ namespace BuildXL {
? r`${qualifier.configuration}/${qualifier.targetRuntime}`
: r`${qualifier.configuration}/${qualifier.targetFramework}/${qualifier.targetRuntime}`,
});
}
}

6
Public/Src/Deployment/tools.dsc
Просмотреть файл

@ -34,7 +34,7 @@ namespace Tools {
});
}
namespace Orchestrator {
namespace AdoBuildRunner {
export declare const qualifier: BuildXLSdk.NetCoreAppQualifier;
@ -42,13 +42,13 @@ namespace Tools {
contents: [
importFrom("BuildXL.Tools").withQualifier({
targetFramework: qualifier.targetFramework,
}).Orchestrator.exe
}).AdoBuildRunner.exe
],
};
const deployed = BuildXLSdk.DeploymentHelpers.deploy({
definition: deployment,
targetLocation: Helpers.getTargetLocation("Orchestrator"),
targetLocation: Helpers.getTargetLocation("AdoBuildRunner"),
});
}

7
Public/Src/Tools/Orchestrator/Orchestrator.dsc → Public/Src/Tools/AdoBuildRunner/AdoBuildRunner.dsc
Просмотреть файл

@ -5,11 +5,11 @@ import { NetFx } from "Sdk.BuildXL";
import * as Managed from "Sdk.Managed";
import * as BuildXLSdk from "Sdk.BuildXL";
namespace Orchestrator {
namespace AdoBuildRunner {
@@public
export const exe = BuildXLSdk.executable({
assemblyName: "Orchestrator",
assemblyName: "AdoBuildRunner",
sources: [
...globR(d`./Build/`, "*.cs"),
...globR(d`./Vsts/`, "*.cs"),
@ -17,12 +17,13 @@ namespace Orchestrator {
f`Program.cs`,
],
references: [
...importFrom("BuildXL.Utilities").Native.securityDlls,
importFrom("Newtonsoft.Json").pkg,
importFrom("Microsoft.AspNet.WebApi.Client").pkg,
importFrom("Microsoft.TeamFoundationServer.Client").pkg,
importFrom("Microsoft.VisualStudio.Services.Client").pkg,
importFrom("Microsoft.TeamFoundation.DistributedTask.WebApi").pkg,
importFrom("Microsoft.TeamFoundation.DistributedTask.Common.Contracts").pkg
importFrom("Microsoft.TeamFoundation.DistributedTask.Common.Contracts").pkg,
],
});
}

2
Public/Src/Tools/Orchestrator/Build/BuildContext.cs → Public/Src/Tools/AdoBuildRunner/Build/BuildContext.cs
Просмотреть файл

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Orchestrator.Build
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// A build context represents an ongoing VSTS build and its most important properties

10
Public/Src/Tools/Orchestrator/Build/BuildExecutor.cs → Public/Src/Tools/AdoBuildRunner/Build/BuildExecutor.cs
Просмотреть файл

@ -5,9 +5,9 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using BuildXL.Orchestrator.Vsts;
using BuildXL.AdoBuildRunner.Vsts;
namespace BuildXL.Orchestrator.Build
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// A build executor that can execute a build engine depending on agent status and build arguments
@ -116,5 +116,11 @@ namespace BuildXL.Orchestrator.Build
buildContext.SourcesDirectory
);
}
/// <inheritdoc />
public void InitializeAsWorker(BuildContext buildContext, string[] buildArguments)
{
// No prep work to do
}
}
}

4
Public/Src/Tools/Orchestrator/Build/BuildExecutorBase.cs → Public/Src/Tools/AdoBuildRunner/Build/BuildExecutorBase.cs
Просмотреть файл

@ -2,9 +2,9 @@
// Licensed under the MIT License.
using System;
using BuildXL.Orchestrator.Vsts;
using BuildXL.AdoBuildRunner.Vsts;
namespace BuildXL.Orchestrator.Build
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// A build executor base class with common properties

45
Public/Src/Tools/Orchestrator/Build/BuildManager.cs → Public/Src/Tools/AdoBuildRunner/Build/BuildManager.cs
Просмотреть файл

@ -6,9 +6,9 @@ using System.Linq;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Threading.Tasks;
using BuildXL.Orchestrator.Vsts;
using BuildXL.AdoBuildRunner.Vsts;
namespace BuildXL.Orchestrator.Build
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// A class managing execution of orchestrated builds depending on VSTS agent states
@ -63,7 +63,7 @@ namespace BuildXL.Orchestrator.Build
m_logger.Info($@"Value of the job position in the phase: {m_vstsApi.JobPositionInPhase}");
m_logger.Info($@"Value of the total jobs in the phase: {m_vstsApi.TotalJobsInPhase}");
var returnCode = 1;
int returnCode;
// Only one agent participating in the build, hence a singe machine build
if (m_vstsApi.TotalJobsInPhase == 1)
@ -74,7 +74,7 @@ namespace BuildXL.Orchestrator.Build
// Currently the agent spawned last in a multi-agent build is the elected master
else if (m_vstsApi.TotalJobsInPhase == m_vstsApi.JobPositionInPhase)
{
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress(), isMaster: true);
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress(false), GetAgentIPAddress(true), isMaster: true);
await m_vstsApi.WaitForOtherWorkersToBeReady();
var machines = (await m_vstsApi.GetWorkerAddressInformationAsync()).ToList();
@ -89,10 +89,11 @@ namespace BuildXL.Orchestrator.Build
// Any agent spawned < total number of agents is a dedicated worker
else
{
await m_vstsApi.WaitForMasterToBeReady();
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress());
m_executor.InitializeAsWorker(buildContext, m_buildArguments);
await m_vstsApi.WaitForMasterToBeReady();
var masterInfo = (await m_vstsApi.GetMasterAddressInformationAsync()).FirstOrDefault();
if (masterInfo == null)
{
throw new ApplicationException($"Couldn't get master address info, aborting!");
@ -100,6 +101,8 @@ namespace BuildXL.Orchestrator.Build
m_logger.Info($@"Found master: {masterInfo[Constants.MachineHostName]}@{masterInfo[Constants.MachineIpV4Address]}");
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress(false), GetAgentIPAddress(true));
returnCode = m_executor.ExecuteDistributedBuildAsWorker(buildContext, m_buildArguments, masterInfo);
LogExitCode(returnCode);
}
@ -112,7 +115,8 @@ namespace BuildXL.Orchestrator.Build
return System.Net.Dns.GetHostName();
}
private string GetAgentIPAddress()
/// <nodoc />
public static string GetAgentIPAddress(bool ipv6)
{
var firstUpInterface = NetworkInterface.GetAllNetworkInterfaces()
.OrderByDescending(c => c.Speed)
@ -121,15 +125,32 @@ namespace BuildXL.Orchestrator.Build
if (firstUpInterface != null)
{
var props = firstUpInterface.GetIPProperties();
// get first IPV4 address assigned to this interface
var ipV4Address = props.UnicastAddresses
if (!ipv6)
{
// get first IPV4 address assigned to this interface
var ipV4Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetwork)
.Select(c => c.Address)
.Select(c => c.Address.ToString())
.FirstOrDefault();
if (ipV4Address != null)
if (ipV4Address != null)
{
return ipV4Address;
}
}
else
{
return ipV4Address.ToString();
var ipV6Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetworkV6)
.Select(c => c.Address.ToString())
.Select(a => a.Split('%').FirstOrDefault() ?? a)
.FirstOrDefault();
if (ipV6Address != null)
{
return ipV6Address;
}
}
}

9
Public/Src/Tools/Orchestrator/Build/IBuildExecutor.cs → Public/Src/Tools/AdoBuildRunner/Build/IBuildExecutor.cs
Просмотреть файл

@ -3,7 +3,7 @@
using System.Collections.Generic;
namespace BuildXL.Orchestrator.Build
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// Defines the interface to execute a distributed build
@ -33,6 +33,13 @@ namespace BuildXL.Orchestrator.Build
/// <returns>Status code of the build argument execution</returns>
int ExecuteDistributedBuildAsMaster(BuildContext buildContext, string[] buildArguments, List<IDictionary<string, string>> workerInfo);
/// <summary>
/// Perfrorm any work before setting the machine "ready" to build
/// </summary>
/// <param name="buildContext">The build context</param>
/// <param name="buildArguments">Arguments to be executed when orchestration succeeds</param>
void InitializeAsWorker(BuildContext buildContext, string[] buildArguments);
/// <summary>
/// Execute a build with a given context and arguments as orchestrated worker
/// </summary>

227
Public/Src/Tools/AdoBuildRunner/Build/PingExecutor.cs Normal file
Просмотреть файл

@ -0,0 +1,227 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Threading.Tasks;
using BuildXL.AdoBuildRunner.Vsts;
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// An executor to diagnose connectivity between hosted agents.
/// This executor will perform a ping and try to establish TCP connections
/// from the orchestrator to the workers, both to their IPV4 and IPV6 addresses
/// </summary>
public class PingExecutor : BuildExecutorBase, IBuildExecutor
{
private const int ListeningPort = 45678;
private TcpListener m_server = null;
/// <nodoc />
public PingExecutor(ILogger logger) : base(logger) { }
/// <inheritdoc />
public void PrepareBuildEnvironment(BuildContext buildContext)
{
if (buildContext == null)
{
throw new ArgumentNullException(nameof(buildContext));
}
}
/// <inherit />
public int ExecuteSingleMachineBuild(BuildContext buildContext, string[] buildArguments)
{
throw new InvalidOperationException("Single machine ping test");
}
/// <inherit />
public int ExecuteDistributedBuildAsMaster(BuildContext buildContext, string[] buildArguments, List<IDictionary<string, string>> machines)
{
Logger.Info($@"Launching ping test as orchestrator");
var usingV6 = buildArguments.Any(opt => opt == "ipv6");
var ip = BuildManager.GetAgentIPAddress(usingV6);
var tasks = new Task<bool>[machines.Count];
for (int i = 0; i < tasks.Length; i++)
{
var workerIp = machines[i][usingV6 ? Constants.MachineIpV6Address : Constants.MachineIpV4Address];
tasks[i] = SendMessageToWorker(ip, machines[0][Constants.MachineHostName], workerIp, usingV6);
}
Task.WhenAll(tasks).GetAwaiter().GetResult();
if (!tasks.All(t => t.GetAwaiter().GetResult()))
{
// Some task failed
return 1;
}
return 0;
}
/// <inherit />
public int ExecuteDistributedBuildAsWorker(BuildContext buildContext, string[] buildArguments, IDictionary<string, string> masterInfo)
{
Logger.Info($@"Launching ping & connectivity test as worker");
WaitMessageFromMaster().GetAwaiter().GetResult();
return 0;
}
private async Task WaitMessageFromMaster()
{
try
{
// Buffer for reading data
var bytes = new byte[256];
string data = null;
// Enter the listening loop.
Logger.Info("Waiting for a connection... ");
TcpClient client = await m_server.AcceptTcpClientAsync();
Logger.Info("Connected!");
data = null;
// Get a stream object for reading and writing
NetworkStream stream = client.GetStream();
int i;
// Loop to receive all the data sent by the client.
while ((i = await stream.ReadAsync(bytes, 0, bytes.Length)) != 0)
{
// Translate data bytes to a ASCII string.
data = System.Text.Encoding.ASCII.GetString(bytes, 0, i);
Logger.Info($"Received: {data}");
// Process the data sent by the client.
data = data.ToUpper();
byte[] msg = System.Text.Encoding.ASCII.GetBytes(data);
// Send back a response.
await stream.WriteAsync(msg, 0, msg.Length);
Logger.Info($"Sent: {data}");
}
// Shutdown and end connection
client.Close();
}
catch (SocketException e)
{
Logger.Info($"SocketException: {e}");
throw;
}
finally
{
// Stop listening for new clients.
m_server.Stop();
}
}
private Task<bool> SendMessageToWorker(string myIp, string workerHostname, string workerIp, bool ipv6)
{
// Ping
try
{
for (var i = 0; i < 2; i++)
{
using var pinger = new Ping();
PingReply reply = pinger.Send(workerIp, timeout: i*1000);
Logger.Info($"Ping to IP {workerIp} response: {reply.Status} - {reply.RoundtripTime}ms");
reply = pinger.Send(workerHostname, timeout: i*1000);
Logger.Info($"Ping to hostname {workerHostname} response: {reply.Status} - {reply.RoundtripTime}ms");
}
}
catch (PingException e)
{
Logger.Info(e.Message);
}
// TCP message
Logger.Info($"Sending a message from {myIp} to {workerIp}:{ListeningPort}");
return SendTcpMessageToWorker(workerIp, ListeningPort, ipv6 ? AddressFamily.InterNetworkV6 : AddressFamily.InterNetwork);
}
private async Task<bool> SendTcpMessageToWorker(string workerIp, int port, AddressFamily addressFamily)
{
int attempts = 0;
TcpClient client = null;
while (++attempts <= 3)
{
Logger.Info($"[-> {workerIp}] Attempt {attempts} of 3");
try
{
client = new TcpClient(addressFamily);
await client.ConnectAsync(IPAddress.Parse(workerIp), port);
}
catch (Exception e)
{
Logger.Info($"[-> {workerIp}] Connect exception: {e}.");
}
if (client.Connected)
{
break;
}
Logger.Info($"[-> {workerIp}] Waiting for 20s before retrying...");
await Task.Delay(20_000);
}
if (!client.Connected)
{
Logger.Error($"[-> {workerIp}] Couldn't connect after 3 attempts");
return false;
}
// Translate the passed message into ASCII and store it as a Byte array.
byte[] data = System.Text.Encoding.ASCII.GetBytes("Hello, world");
// Get a client stream for reading and writing.
NetworkStream stream = client.GetStream();
// Send the message to the connected TcpServer.
await stream.WriteAsync(data, 0, data.Length);
await stream.FlushAsync();
Logger.Info($"[-> {workerIp}] Sent: Hello, world");
// Receive the TcpServer.response.
// Buffer to store the response bytes.
data = new byte[256];
// String to store the response ASCII representation.
string responseData = string.Empty;
// Read the first batch of the TcpServer response bytes.
int bytes = await stream.ReadAsync(data, 0, data.Length);
responseData = System.Text.Encoding.ASCII.GetString(data, 0, bytes);
Logger.Info($"[-> {workerIp}] Received: {responseData}");
// Close everything.
stream.Close();
client.Close();
return true;
}
/// <inheritdoc />
public void InitializeAsWorker(BuildContext buildContext, string[] buildArguments)
{
// Start listening for client requests.
m_server = TcpListener.Create(ListeningPort);
m_server.Start();
Logger.Info($"Started server on port {ListeningPort}");
}
}
}

20
Public/Src/Tools/Orchestrator/Constants.cs → Public/Src/Tools/AdoBuildRunner/Constants.cs
Просмотреть файл

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Orchestrator
namespace BuildXL.AdoBuildRunner
{
/// <summary>
/// Collection of constants used for orchestration, mostly VSTS agent environment variables
@ -74,9 +74,9 @@ namespace BuildXL.Orchestrator
public const string RepositoryUrlVariableName = "BUILD_REPOSITORY_URI";
/// <summary>
/// The name of the VSTS orchestration task
/// Name of the variable of the current task's display name
/// </summary>
public const string BuildOrchestrationTaskName = "VSTSBuildOrchestrationTask";
public const string TaskDisplayNameVariableName = "SYSTEM_TASKDISPLAYNAME";
/// <summary>
/// Variable indicating the current agent type
@ -89,10 +89,15 @@ namespace BuildXL.Orchestrator
public const string MachineHostName = "MachineHostName";
/// <summary>
/// Name of the task variable used to determine if the machine is ready to build
/// Name of the task variable used to communicate the machine IPV4 address
/// </summary>
public const string MachineIpV4Address = "MachineIpV4Address";
/// <summary>
/// Name of the task variable used to communicate the machine IPV4 address
/// </summary>
public const string MachineIpV6Address = "MachineIpV6Address";
/// <summary>
/// The port used to establish GRPC based connections for distributed builds
/// </summary>
@ -101,7 +106,12 @@ namespace BuildXL.Orchestrator
/// <summary>
/// The maximum time an agent waits for the other agents to get ready before failing
/// </summary>
public const int MaxWaitingPeriodBeforeFailingInSeconds = 600;
public const int DefaultMaximumWaitForWorkerSeconds = 600;
/// <summary>
/// The maximum time an agent waits for the other agents to get ready before failing
/// </summary>
public const string MaximumWaitForWorkerSecondsVariableName = "MaximumWaitForWorkerSeconds";
/// <summary>
/// The time the agent waits before re-checking if the other agents are ready

21
Public/Src/Tools/Orchestrator/Program.cs → Public/Src/Tools/AdoBuildRunner/Program.cs
Просмотреть файл

@ -2,10 +2,10 @@
// Licensed under the MIT License.
using System;
using BuildXL.Orchestrator.Build;
using BuildXL.Orchestrator.Vsts;
using BuildXL.AdoBuildRunner.Build;
using BuildXL.AdoBuildRunner.Vsts;
namespace BuildXL.Orchestrator
namespace BuildXL.AdoBuildRunner
{
class Program
{
@ -21,8 +21,19 @@ namespace BuildXL.Orchestrator
try
{
logger.Info($"Trying to orchestrate build for command: {string.Join(" ", args)}");
var buildManager = new BuildManager(new Api(logger), new BuildExecutor(logger), args, logger);
IBuildExecutor executor;
if (args[0] == "ping")
{
logger.Info("Performing connectivity test");
executor = new PingExecutor(logger);
}
else
{
logger.Info($"Trying to orchestrate build for command: {string.Join(" ", args)}");
executor = new BuildExecutor(logger);
}
var buildManager = new BuildManager(new Api(logger), executor, args, logger);
return buildManager.BuildAsync().GetAwaiter().GetResult();
}
catch (Exception e)

58
Public/Src/Tools/Orchestrator/Vsts/Api.cs → Public/Src/Tools/AdoBuildRunner/Vsts/Api.cs
Просмотреть файл

@ -10,7 +10,7 @@ using Microsoft.TeamFoundation.DistributedTask.WebApi;
using Microsoft.VisualStudio.Services.Common;
using TimelineRecord = Microsoft.TeamFoundation.DistributedTask.WebApi.TimelineRecord;
namespace BuildXL.Orchestrator.Vsts
namespace BuildXL.AdoBuildRunner.Vsts
{
/// <summary>
/// Concrete implementation of the VSTS API interface for build orchestration purposes
@ -31,6 +31,9 @@ namespace BuildXL.Orchestrator.Vsts
private const string HubType = "build";
// Timeouts
private readonly int m_maxWaitingTimeSeconds;
/// <nodoc />
public string BuildId { get; }
@ -114,6 +117,25 @@ namespace BuildXL.Orchestrator.Vsts
m_taskClient = new TaskHttpClient(server, cred);
m_buildClient = new BuildHttpClient(server, cred);
m_maxWaitingTimeSeconds = Constants.DefaultMaximumWaitForWorkerSeconds;
var userMaxWaitingTime = Environment.GetEnvironmentVariable(Constants.MaximumWaitForWorkerSecondsVariableName);
if (!string.IsNullOrEmpty(userMaxWaitingTime))
{
if (!int.TryParse(userMaxWaitingTime, out var maxWaitingTime))
{
m_logger.Warning($"Couldn't parse value '{userMaxWaitingTime}' for {Constants.MaximumWaitForWorkerSecondsVariableName}." +
$"Using the default value of {Constants.DefaultMaximumWaitForWorkerSeconds}");
}
else
{
m_maxWaitingTimeSeconds = maxWaitingTime;
}
}
m_maxWaitingTimeSeconds = string.IsNullOrEmpty(userMaxWaitingTime) ?
Constants.DefaultMaximumWaitForWorkerSeconds
: int.Parse(userMaxWaitingTime);
}
private async Task<IEnumerable<IDictionary<string, string>>> GetAddressInformationAsync(AgentType type)
@ -142,11 +164,15 @@ namespace BuildXL.Orchestrator.Vsts
private async Task<List<TimelineRecord>> GetTimelineRecords()
{
List<TimelineRecord> records = await m_taskClient.GetRecordsAsync(new Guid(TeamProjectId), HubType, new Guid(PlanId), new Guid(TimelineId));
List<TimelineRecord> timelineRecords =
records.Where(r => r.Name.Equals(Constants.BuildOrchestrationTaskName, StringComparison.OrdinalIgnoreCase)).ToList();
var currentTask = Environment.GetEnvironmentVariable(Constants.TaskDisplayNameVariableName);
return timelineRecords;
m_logger.Debug($"Getting timeline records for task '{currentTask}'");
var allRecords = await m_taskClient.GetRecordsAsync(new Guid(TeamProjectId), HubType, new Guid(PlanId), new Guid(TimelineId));
var records = allRecords.Where(r => r.Name == currentTask).ToList();
m_logger.Debug($"Found {records.Count} records");
return records;
}
/// <inherit />
@ -162,17 +188,17 @@ namespace BuildXL.Orchestrator.Vsts
}
/// <inherit />
public async Task SetMachineReadyToBuild(string hostName, string ipV4Address, bool isMaster)
public async Task SetMachineReadyToBuild(string hostName, string ipV4Address, string ipv6Address, bool isMaster)
{
List<TimelineRecord> records = await GetTimelineRecords();
// Inject the information into a timeline record for this worker
var records = await GetTimelineRecords();
TimelineRecord record = records.FirstOrDefault(t => t.WorkerName.Equals(AgentName, StringComparison.OrdinalIgnoreCase));
if (record != null)
{
// Add / update agent info for the build orchestration
record.Variables[Constants.MachineType] = (isMaster ? AgentType.Master : AgentType.Worker).ToString();
record.Variables[Constants.MachineHostName] = hostName;
record.Variables[Constants.MachineIpV4Address] = ipV4Address;
record.Variables[Constants.MachineIpV6Address] = ipv6Address;
await m_taskClient.UpdateTimelineRecordsAsync(
new Guid(TeamProjectId),
@ -180,6 +206,12 @@ namespace BuildXL.Orchestrator.Vsts
new Guid(PlanId),
new Guid(TimelineId),
new List<TimelineRecord>() { record });
m_logger.Info("Marked machine as ready to build in the timeline records");
}
else
{
throw new ApplicationException("No records found for this worker");
}
}
@ -202,10 +234,12 @@ namespace BuildXL.Orchestrator.Vsts
var otherAgentsAreReady = false;
var elapsedTime = 0;
while (!otherAgentsAreReady && elapsedTime < Constants.MaxWaitingPeriodBeforeFailingInSeconds)
while (!otherAgentsAreReady && elapsedTime < m_maxWaitingTimeSeconds)
{
List<TimelineRecord> records = await GetTimelineRecords();
if (records.Any(r => r.ErrorCount.HasValue && r.ErrorCount.Value != 0))
var errors = records.Where(r => r.ErrorCount.HasValue && r.ErrorCount.Value != 0);
if (errors.Any())
{
throw new ApplicationException("One of the agents failed during the orchestration task with errors, aborting build!");
}
@ -235,7 +269,7 @@ namespace BuildXL.Orchestrator.Vsts
}
}
if (elapsedTime >= Constants.MaxWaitingPeriodBeforeFailingInSeconds)
if (elapsedTime >= m_maxWaitingTimeSeconds)
{
throw new ApplicationException("Waiting for all agents to get ready failed, aborting!");
}

4
Public/Src/Tools/Orchestrator/Vsts/IApi.cs → Public/Src/Tools/AdoBuildRunner/Vsts/IApi.cs
Просмотреть файл

@ -5,7 +5,7 @@ using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace BuildXL.Orchestrator.Vsts
namespace BuildXL.AdoBuildRunner.Vsts
{
/// <summary>
/// Defines the interactions with the VSTS API
@ -94,7 +94,7 @@ namespace BuildXL.Orchestrator.Vsts
/// Indicate that this machine is ready to build using a timeline record
/// </summary>
/// <returns></returns>
Task SetMachineReadyToBuild(string hostName, string ipV4Address, bool isMaster = false);
Task SetMachineReadyToBuild(string hostName, string ipV4Address, string ipv6Address, bool isMaster = false);
/// <summary>
/// Wait until all the other workers are ready

2
Public/Src/Tools/Orchestrator/Vsts/ILogger.cs → Public/Src/Tools/AdoBuildRunner/Vsts/ILogger.cs
Просмотреть файл

@ -3,7 +3,7 @@
using System;
namespace BuildXL.Orchestrator.Vsts
namespace BuildXL.AdoBuildRunner.Vsts
{
/// <summary>
/// Interface that defines the methods that must be implemented by VSTS loggers

12
Public/Src/Tools/Orchestrator/Vsts/Logger.cs → Public/Src/Tools/AdoBuildRunner/Vsts/Logger.cs
Просмотреть файл

@ -4,7 +4,7 @@
using System;
using Newtonsoft.Json;
namespace BuildXL.Orchestrator.Vsts
namespace BuildXL.AdoBuildRunner.Vsts
{
/// <summary>
/// This is a special logger for VSTS task using the special vso logging commands ##vso
@ -29,7 +29,7 @@ namespace BuildXL.Orchestrator.Vsts
/// <nodoc />
public void Error(string message)
{
string errorCommand = string.Format(LogErrorFormat, message);
string errorCommand = string.Format(LogErrorFormat, WithTimeStamp(message));
Console.WriteLine(errorCommand);
}
@ -71,21 +71,23 @@ namespace BuildXL.Orchestrator.Vsts
/// <nodoc />
public void Warning(string message)
{
string warningCommand = string.Format(LogWarningFormat, message);
string warningCommand = string.Format(LogWarningFormat, WithTimeStamp(message));
Console.WriteLine(warningCommand);
}
/// <nodoc />
public void Debug(string message)
{
string debugCommand = string.Format(LogDebugFormat, message);
string debugCommand = string.Format(LogDebugFormat, WithTimeStamp(message));
Console.WriteLine(debugCommand);
}
/// <nodoc />
public void Info(string message)
{
Console.WriteLine(message);
Console.WriteLine(WithTimeStamp(message));
}
private string WithTimeStamp(string message) => string.Format("[{0}] {1}", DateTime.UtcNow.ToString("HH:mm:ss.ff"), message);
}
}

13
Public/Src/Tools/Orchestrator/Orchestrator.csproj
Просмотреть файл

@ -1,13 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<PlatformTarget>x64</PlatformTarget>
<TargetFramework>netcoreapp3.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.7" />
<PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="16.146.0-preview" />
<PackageReference Include="Microsoft.VisualStudio.Services.Client" Version="16.146.0-preview" />
<PackageReference Include="Microsoft.TeamFoundation.DistributedTask.WebApi" Version="16.146.0-preview" />
</ItemGroup>
</Project>

15
cg/nuget/cgmanifest.json
Просмотреть файл

@ -2125,21 +2125,12 @@
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.TeamFoundation.DistributedTask.Common",
"Version": "15.112.1"
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.TeamFoundation.DistributedTask.Common.Contracts",
"Version": "16.137.0-preview"
"Version": "16.170.0"
}
}
},
@ -2148,7 +2139,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.TeamFoundation.DistributedTask.WebApi",
"Version": "15.122.1-preview"
"Version": "16.170.0"
}
}
},
@ -2157,7 +2148,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "Microsoft.TeamFoundationServer.Client",
"Version": "15.122.1-preview"
"Version": "16.170.0"
}
}
},

7
config.dsc
Просмотреть файл

@ -298,11 +298,10 @@ config({
{ id: "Validation", version: "2.5.42"},
// VSTS managed API
{ id: "Microsoft.TeamFoundationServer.Client", version: "15.122.1-preview"},
{ id: "Microsoft.TeamFoundation.DistributedTask.WebApi", version: "15.122.1-preview",
{ id: "Microsoft.TeamFoundationServer.Client", version: "16.170.0"},
{ id: "Microsoft.TeamFoundation.DistributedTask.WebApi", version: "16.170.0",
dependentPackageIdsToSkip: ["*"] },
{ id: "Microsoft.TeamFoundation.DistributedTask.Common", version: "15.112.1"},
{ id: "Microsoft.TeamFoundation.DistributedTask.Common.Contracts", version: "16.137.0-preview"},
{ id: "Microsoft.TeamFoundation.DistributedTask.Common.Contracts", version: "16.170.0"},
// MSBuild. These should be used for compile references only, as at runtime one can only practically use MSBuilds from Visual Studio / dotnet CLI
{ id: "Microsoft.Build", version: "17.0.0",