microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 744993: Consolidate BuildManagers in AdoBuildRunner 

As of today we have two distinct ways of running distributed builds on ADO: the original model, where all build agents run the same job and are multiplied with the parallel strategy, and the worker pipeline model, where a second pipeline is triggered where all agents run as workers, and a single agent runs a build in the original pipeline (as an orchestrator).

The two approaches were using different ways to coordinate the distributed build (namely, communicating the relevant build information such as build id and orchestrator location), so we had two "BuildManager" classes which were instantiated for the different scenarios. But the fact is that we can use the same pre-build coordination in both scenarios: this PR consolidates this into a single `BuildManager` class which corresponds to the up-until-now called `WorkerPipelineBuildManager`. This communicates the information via the build properties of the orchestrator build (which for the "parallel agents" scenario is the exact same build as the workers, but for the "worker pipeline" scenario).

Note that the "parallel agents" scenario is only used internally for our selfhost validations. The PR includes the only change needed in the Linux validation YAML to accommodate this consolidation (passing an invocation key is now required).
This commit is contained in:
Marcelo Lynch 🧉 2023-10-14 02:53:55 +00:00
Родитель 61dedb5385
Коммит 0914b01f39
8 изменённых файлов: 107 добавлений и 462 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

104
.azdo/distributed/selfhost-clean-main.yml
Просмотреть файл

@ -1,104 +0,0 @@
# This pipeline does a clean build of the BuildXL repo as a distributed build
# The BuildXL version to download from drop and use for the build (e.g., 0.1.0-20221026.0)
# should be specified via the queue-time variable $(BuildXLPackageVersion)
trigger: none # This pipeline is explicitly scheduled
parameters:
- name: DropName
type: string
default: buildxl.dogfood.$(BuildXLPackageVersion)
- name: WorkerPipelineId
type: string
variables:
- group: "BuildXL Common variables"
- group: "BuildXL Secrets"
- name: Domino.DogfoodPackagePath
value: $(Build.StagingDirectory)\$(Build.BuildId)\Dogfood
- name: BuildXL.RepoDirectory
value: $(Build.Repository.LocalPath)
- name: BuildXL.LogsDirectory
value: $(BuildXL.RepoDirectory)\Out\Logs
- name: PatArgs
value: -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
- name: BxlScriptArgs
value: -UseAdoBuildRunner -UseEphemeralCache Build -SharedCacheMode ConsumeAndPublish -Use Dev -DevRoot $(Domino.DogfoodPackagePath)\release\win-x64
- name: BuildXLArgs
value: /q:ReleaseDotNet6 /server- /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=* /p:BuildXLWorkerAttachTimeoutMin=5 /logOutput:FullOutputOnWarningOrError /p:RetryXunitTests=1 /processRetries:3 /traceinfo:valdation=ReleasePipelineDistribution /enableIncrementalFrontEnd- /p:xunitSemaphoreCount=12
pool:
name: BuildXL-DevOpsAgents-PME
jobs:
- job: Distributed_Clean
steps:
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@1
displayName: 'Download ${{ parameters.DropName }} from drop'
inputs:
dropServiceURI: 'https://mseng.artifacts.visualstudio.com/DefaultCollection'
buildNumber: '${{ parameters.DropName }}'
destinationPath: '$(Domino.DogfoodPackagePath)'
rootPaths: 'release/win-x64'
- task: PowerShell@2
inputs:
targetType: 'inline'
script: .\AdoBuildRunner.exe launchworkers ${{ parameters.WorkerPipelineId }} /param:DropName=${{ parameters.DropName }}
workingDirectory: $(Domino.DogfoodPackagePath)\release\win-x64
displayName: 'Trigger worker pipeline'
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- checkout: self
fetchDepth: 1
- template: ../common/set-artifacts-pat.yml
parameters:
AzDevPAT: $(PAT-TseBuild-AzureDevOps-MsEng)
- template: ../common/journaling.yml
- script: |
netsh advfirewall firewall add rule name="Open BuildXL inbound port" dir=in action=allow protocol=TCP localport=6979
netsh advfirewall firewall add rule name="Open BuildXL outbound port" protocol=TCP localport=6979 action=allow dir=OUT
netsh advfirewall firewall add rule name="Open Cache inbound port" dir=in action=allow protocol=TCP localport=7089
netsh advfirewall firewall add rule name="Open Cache outbound port" protocol=TCP localport=7089 action=allow dir=OUT
displayName: Setup firewall rules
- task: PowerShell@2
inputs:
targetType: filePath
filePath: 'RunBxlWithPAT.ps1'
# CODESYNC: /dynamicBuildWorkerSlots must be kept in sync with the amount of workers in the worker pipeline
arguments: '$(PatArgs) $(BxlScriptArgs) /dynamicBuildWorkerSlots:2 $(BuildXLArgs) /logsDirectory:$(BuildXL.LogsDirectory) /ado'
workingDirectory: $(BuildXL.RepoDirectory)
displayName: 'Run clean selfhost build distributed'
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
AdoBuildRunnerWorkerPipelineRole: Orchestrator # TODO: This will be removed after we fully transition to the worker-pipeline model
BlobCacheFactoryConnectionString: $(BuildXL-Selfhost-L3-ConnectionString)
- powershell: |
$bxlWithRemotingLogDir = "$(BuildXL.LogsDirectory)"
$statsContent = Get-Content -Path (Join-Path $bxlWithRemotingLogDir "BuildXL.stats") | Out-String
$stats = ConvertFrom-StringData -StringData $statsContent
$runDistributedCount = $stats.Get_Item("PipExecution.ProcessesExecutedRemotely")
Write-Host "Pips run on remote workers: $runDistributedCount"
if ($runDistributedCount -eq 0)
{
Write-Error "##[error]No process pip ran in distributed workers."
exit 1
}
exit 0
displayName: 'Validate that distribution happened'
condition: succeeded()
- task: PublishPipelineArtifact@1
displayName: 'Upload logs'
inputs:
targetPath: $(BuildXL.LogsDirectory)
condition: always()
displayName: Clean distributed build

76
.azdo/distributed/selfhost-clean-workers.yml
Просмотреть файл

@ -1,76 +0,0 @@
# Worker pipeline
trigger: none # This pipeline is explicitly scheduled
variables:
- group: "BuildXL Common variables"
- group: "BuildXL Secrets"
- name: Domino.DogfoodPackagePath
value: $(Build.StagingDirectory)\$(Build.BuildId)\Dogfood
- name: BuildXL.RepoDirectory
value: $(Build.Repository.LocalPath)
- name: BuildXL.LogsDirectory
value: $(BuildXL.RepoDirectory)\Out\Logs
- name: PatArgs
value: -OneEsPat $(PAT-TseBuild-AzureDevOps-1esSharedAssets-Package-Read) -CbPat $(PAT-TseBuild-AzureDevOps-CloudBuild-Packaging-Read) -MsEngGitPat $(PAT-TseBuild-AzureDevOps-MsEng-ReadCode) -VstsPat $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
- name: BxlScriptArgs
value: -UseAdoBuildRunner -UseEphemeralCache Build -SharedCacheMode ConsumeAndPublish -Use Dev -DevRoot $(Domino.DogfoodPackagePath)\release\win-x64
- name: BuildXLArgs
value: /q:ReleaseDotNet6 /server- /p:[Sdk.BuildXL]microsoftInternal=1 /p:BUILDXL_FINGERPRINT_SALT=* /p:BuildXLWorkerAttachTimeoutMin=5 /logOutput:FullOutputOnWarningOrError /p:RetryXunitTests=1 /processRetries:3 /traceinfo:valdation=ReleasePipelineDistribution /enableIncrementalFrontEnd- /p:xunitSemaphoreCount=12
parameters:
- name: DropName
type: string
pool:
name: BuildXL-DevOpsAgents-PME
jobs:
- job: Distributed_Clean_Workers
strategy:
parallel: 2 # CODESYNC: this value must be kept in sync with /dynamicBuildWorkerSlot in the worker pipeline
displayName: Clean distributed build (workers)
steps:
- task: ms-vscs-artifact.build-tasks.artifactDropDownloadTask-1.artifactDropDownloadTask@1
displayName: 'Download ${{ parameters.DropName }} from drop'
inputs:
dropServiceURI: 'https://mseng.artifacts.visualstudio.com/DefaultCollection'
buildNumber: '${{ parameters.DropName }}'
destinationPath: '$(Domino.DogfoodPackagePath)'
rootPaths: 'release/win-x64'
- checkout: self
fetchDepth: 1
- template: ../common/set-artifacts-pat.yml
parameters:
AzDevPAT: $(PAT-TseBuild-AzureDevOps-MsEng)
- template: ../common/journaling.yml
- template: ../common/set-msvc-version.yml
- script: |
netsh advfirewall firewall add rule name="Open BuildXL inbound port" dir=in action=allow protocol=TCP localport=6979
netsh advfirewall firewall add rule name="Open BuildXL outbound port" protocol=TCP localport=6979 action=allow dir=OUT
netsh advfirewall firewall add rule name="Open Cache inbound port" dir=in action=allow protocol=TCP localport=7089
netsh advfirewall firewall add rule name="Open Cache outbound port" protocol=TCP localport=7089 action=allow dir=OUT
displayName: Setup firewall rules
- task: PowerShell@2
inputs:
targetType: filePath
filePath: 'RunBxlWithPAT.ps1'
arguments: '$(PatArgs) $(BxlScriptArgs) $(BuildXLArgs) /logsDirectory:$(BuildXL.LogsDirectory) /ado'
workingDirectory: $(BuildXL.RepoDirectory)
displayName: 'Run clean selfhost build distributed'
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
AdoBuildRunnerWorkerPipelineRole: Worker # TODO: This will be removed after we fully transition to the worker-pipeline model
BlobCacheFactoryConnectionString: $(BuildXL-Selfhost-L3-ConnectionString)
- task: PublishPipelineArtifact@1
displayName: 'Upload logs'
inputs:
targetPath: $(BuildXL.LogsDirectory)
condition: always()

1
.azdo/linux/job-selfhost.yml
Просмотреть файл

@ -158,6 +158,7 @@ jobs:
VSTSPERSONALACCESSTOKEN: $(PAT-TseBuild-AzureDevOps-mseng-buildcache)
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
AdoBuildRunnerWaitForOrchestratorExit: true
AdoBuildRunnerInvocationKey: LinuxSelfhostValidation_${{ parameters.validationName }}
- task: PublishTestResults@2
displayName: Publish Test Results

184
Public/Src/Tools/AdoBuildRunner/Build/BuildManager.cs
Просмотреть файл

@ -1,16 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.Linq;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using AdoBuildRunner.Vsts;
using BuildXL.AdoBuildRunner.Vsts;
#nullable enable
@ -25,9 +17,9 @@ namespace BuildXL.AdoBuildRunner.Build
private readonly IApi m_vstsApi;
private readonly IBuildExecutor m_executor;
private readonly BuildContext m_buildContext;
private readonly string[] m_buildArguments;
private readonly string[] m_buildArguments;
private readonly BuildContext m_buildContext;
private readonly ILogger m_logger;
/// <summary>
@ -36,126 +28,58 @@ namespace BuildXL.AdoBuildRunner.Build
/// </summary>
/// <param name="vstsApi">Interface to interact with VSTS API</param>
/// <param name="executor">Interface to execute the build engine</param>
/// <param name="buildContext">The build context</param>
/// <param name="args">Build CLI arguments</param>
/// <param name="buildContext">Build context</param>
/// <param name="logger">Interface to log build info</param>
public BuildManager(IApi vstsApi, IBuildExecutor executor, BuildContext buildContext, string[] args, ILogger logger)
{
m_vstsApi = vstsApi;
m_executor = executor;
m_buildContext = buildContext;
m_logger = logger;
m_buildArguments = args;
m_buildContext = buildContext;
}
/// <summary>
/// Computes a related session id that will be consistent across the jobs in the parallel builds run by this manager
/// </summary>
private string ComputeRelatedSessionId()
{
string attemptNumber = Environment.GetEnvironmentVariable(Constants.JobAttemptVariableName)!;
string startTime = m_buildContext.StartTime.ToString("MMdd_HHmmss");
var r = GuidFromString($"{m_buildContext.TeamProjectId}-{m_buildContext.BuildId}-{startTime}-{attemptNumber}");
m_logger.Info($"Computed related session id for this build: {r}");
return r;
}
#pragma warning disable CA5350 // GuidFromString uses a weak cryptographic algorithm SHA1.
private static string GuidFromString(string value)
{
using var hash = SHA1.Create();
byte[] bytesToHash = Encoding.Unicode.GetBytes(value);
hash.TransformFinalBlock(bytesToHash, 0, bytesToHash.Length);
Contract.Assert(hash.Hash is not null);
// Guid takes a 16-byte array
byte[] low16 = new byte[16];
Array.Copy(hash.Hash, low16, 16);
return new Guid(low16).ToString("D");
}
#pragma warning restore CA5350 // GuidFromString uses a weak cryptographic algorithm SHA1.
/// <summary>
/// Executes a build depending on orchestrator / worker context
/// </summary>
/// <returns>The exit code returned by the worker process</returns>
public async Task<int> BuildAsync()
public async Task<int> BuildAsync(bool isOrchestrator)
{
// Possibly extend context with additional info that can influence the build environment as needed
m_executor.PrepareBuildEnvironment(m_buildContext);
m_logger.Info($@"Value of the job position in the phase: {m_vstsApi.JobPositionInPhase}");
m_logger.Info($@"Value of the total jobs in the phase: {m_vstsApi.TotalJobsInPhase}");
int returnCode;
// Only one agent participating in the build, hence a singe machine build
if (m_vstsApi.TotalJobsInPhase == 1)
if (isOrchestrator)
{
returnCode = m_executor.ExecuteSingleMachineBuild(m_buildContext, m_buildArguments);
PublishRoleInEnvironment(isOrchestrator: true);
LogExitCode(returnCode);
// The orchestrator creates the build info and publishes it to the build properties
var buildInfo = new BuildInfo { RelatedSessionId = Guid.NewGuid().ToString("D"), OrchestratorLocation = m_buildContext.AgentMachineName };
await m_vstsApi.PublishBuildInfo(m_buildContext, buildInfo);
returnCode = m_executor.ExecuteDistributedBuildAsOrchestrator(m_buildContext, buildInfo.RelatedSessionId, m_buildArguments);
}
// The first agent spawned in a multi-agent build is the elected orchestrator
else if (m_vstsApi.JobPositionInPhase == 1)
{
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress(false), GetAgentIPAddress(true), isOrchestrator: true);
// Add the dynamic slot argument. In the worker pool approach this is defined
// externally, here we infer it from the amount of machines in this parallel execution
var numDynamicWorkers = m_vstsApi.TotalJobsInPhase - 1;
var buildArguments = new List<string>(m_buildArguments)
{
$"/dynamicBuildWorkerSlots:{numDynamicWorkers}"
};
returnCode = m_executor.ExecuteDistributedBuildAsOrchestrator(m_buildContext, ComputeRelatedSessionId(), buildArguments.ToArray());
await m_vstsApi.SetBuildResult(success: returnCode == 0);
PublishRoleInEnvironment(isOrchestrator: true);
LogExitCode(returnCode);
}
// Any agent spawned < total number of agents is a dedicated worker
else
{
m_executor.InitializeAsWorker(m_buildContext, m_buildArguments);
await m_vstsApi.WaitForOrchestratorToBeReady();
var orchestratorInfo = (await m_vstsApi.GetOrchestratorAddressInformationAsync()).FirstOrDefault();
if (orchestratorInfo == null)
{
CoordinationException.LogAndThrow(m_logger, "Couldn't get orchestrator address info, aborting!");
}
m_logger.Info($@"Found orchestrator: {orchestratorInfo![Constants.MachineHostName]}@{orchestratorInfo[Constants.MachineIpV4Address]}");
await m_vstsApi.SetMachineReadyToBuild(GetAgentHostName(), GetAgentIPAddress(false), GetAgentIPAddress(true));
var buildInfo = new BuildInfo
{
OrchestratorLocation = $"{orchestratorInfo[Constants.MachineHostName]}",
RelatedSessionId = ComputeRelatedSessionId()
};
// Get the build info from the orchestrator build
var buildInfo = await m_vstsApi.WaitForBuildInfo(m_buildContext);
returnCode = m_executor.ExecuteDistributedBuildAsWorker(m_buildContext, buildInfo, m_buildArguments);
LogExitCode(returnCode);
PublishRoleInEnvironment(isOrchestrator: false);
}
LogExitCode(returnCode);
if (!isOrchestrator && Environment.GetEnvironmentVariable(Constants.WaitForOrchestratorExitVariableName) == "true")
{
// If the worker finished successfully but the build fails, we may still want to fail this task
// so the task can be retried as a distributed build with the same number of workers by
// running "retry failed tasks". This behavior makes the agents wait unnecessarily so it's
// disabled by default.
if (returnCode == 0)
{
// If the worker finished successfully but the build fails, we may still want to fail this task
// so the task can be retried as a distributed build with the same number of workers by
// running "retry failed tasks". This behavior makes the agents wait unnecessarily so it's
// disabled by default.
if (Environment.GetEnvironmentVariable(Constants.WaitForOrchestratorExitVariableName) == "true")
var orchestratorSucceeded = await m_vstsApi.WaitForOrchestratorExit();
if (!orchestratorSucceeded)
{
var orchestratorSucceeded = await m_vstsApi.WaitForOrchestratorExit();
if (!orchestratorSucceeded)
{
m_logger.Error($"The build finished with errors in the orchestrator. Failing this task with exit code {Constants.OrchestratorFailedWorkerReturnCode} so this worker will participate in retries.");
returnCode = Constants.OrchestratorFailedWorkerReturnCode;
}
m_logger.Error($"The build finished with errors in the orchestrator. Failing this task with exit code {Constants.OrchestratorFailedWorkerReturnCode} so this worker will participate in retries.");
returnCode = Constants.OrchestratorFailedWorkerReturnCode;
}
}
else
@ -171,62 +95,6 @@ namespace BuildXL.AdoBuildRunner.Build
return returnCode;
}
// Some post-build steps in the job may be interested in which build role
// was adopted by this agent. We expose this fact through an environment variable
private void PublishRoleInEnvironment(bool isOrchestrator)
{
var role = isOrchestrator ? Constants.BuildRoleOrchestrator : Constants.BuildRoleWorker;
m_logger.Info($"Setting environment variable {Constants.BuildRoleVariableName}={role}");
Console.WriteLine($"##vso[task.setvariable variable={Constants.BuildRoleVariableName}]{role}");
}
private string GetAgentHostName()
{
return System.Net.Dns.GetHostName();
}
/// <nodoc />
public static string GetAgentIPAddress(bool ipv6)
{
var firstUpInterface = NetworkInterface.GetAllNetworkInterfaces()
.OrderByDescending(c => c.Speed)
.FirstOrDefault(c => c.NetworkInterfaceType != NetworkInterfaceType.Loopback && c.OperationalStatus == OperationalStatus.Up);
if (firstUpInterface != null)
{
var props = firstUpInterface.GetIPProperties();
if (!ipv6)
{
// get first IPV4 address assigned to this interface
var ipV4Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetwork)
.Select(c => c.Address.ToString())
.FirstOrDefault();
if (ipV4Address != null)
{
return ipV4Address;
}
}
else
{
var ipV6Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetworkV6)
.Select(c => c.Address.ToString())
.Select(a => a.Split('%').FirstOrDefault() ?? a)
.FirstOrDefault();
if (ipV6Address != null)
{
return ipV6Address;
}
}
}
throw new ApplicationException($"Unable to determine IP address, aborting!");
}
private void LogExitCode(int returnCode)
{
if (returnCode != 0)

43
Public/Src/Tools/AdoBuildRunner/Build/PingExecutor.cs
Просмотреть файл

@ -52,7 +52,7 @@ namespace BuildXL.AdoBuildRunner.Build
Logger.Info($@"Launching ping test as orchestrator");
var usingV6 = buildArguments.Any(opt => opt == "ipv6");
var ip = BuildManager.GetAgentIPAddress(usingV6);
var ip = GetAgentIPAddress(usingV6);
var tasks = new Task<bool>[machines.Count];
for (int i = 0; i < tasks.Length; i++)
@ -229,5 +229,46 @@ namespace BuildXL.AdoBuildRunner.Build
m_server.Start();
Logger.Info($"Started server on port {ListeningPort}");
}
private static string GetAgentIPAddress(bool ipv6)
{
var firstUpInterface = NetworkInterface.GetAllNetworkInterfaces()
.OrderByDescending(c => c.Speed)
.FirstOrDefault(c => c.NetworkInterfaceType != NetworkInterfaceType.Loopback && c.OperationalStatus == OperationalStatus.Up);
if (firstUpInterface != null)
{
var props = firstUpInterface.GetIPProperties();
if (!ipv6)
{
// get first IPV4 address assigned to this interface
var ipV4Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetwork)
.Select(c => c.Address.ToString())
.FirstOrDefault();
if (ipV4Address != null)
{
return ipV4Address;
}
}
else
{
var ipV6Address = props.UnicastAddresses
.Where(c => c.Address.AddressFamily == AddressFamily.InterNetworkV6)
.Select(c => c.Address.ToString())
.Select(a => a.Split('%').FirstOrDefault() ?? a)
.FirstOrDefault();
if (ipV6Address != null)
{
return ipV6Address;
}
}
}
throw new ApplicationException($"Unable to determine IP address, aborting!");
}
}
}

84
Public/Src/Tools/AdoBuildRunner/Build/WorkerPipelineBuildManager.cs
Просмотреть файл

@ -1,84 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Threading.Tasks;
using BuildXL.AdoBuildRunner.Vsts;
#nullable enable
namespace BuildXL.AdoBuildRunner.Build
{
/// <summary>
/// A class managing execution of orchestrated builds depending on VSTS agent states
/// </summary>
public class WorkerPipelineBuildManager
{
private readonly IApi m_vstsApi;
private readonly IBuildExecutor m_executor;
private readonly string[] m_buildArguments;
private readonly BuildContext m_buildContext;
private readonly ILogger m_logger;
/// <summary>
/// Initializes the build manager with a concrete VSTS API implementation and all parameters necessary
/// to orchestrate a distributed build
/// </summary>
/// <param name="vstsApi">Interface to interact with VSTS API</param>
/// <param name="executor">Interface to execute the build engine</param>
/// <param name="args">Build CLI arguments</param>
/// <param name="buildContext">Build context</param>
/// <param name="logger">Interface to log build info</param>
public WorkerPipelineBuildManager(IApi vstsApi, IBuildExecutor executor, BuildContext buildContext, string[] args, ILogger logger)
{
m_vstsApi = vstsApi;
m_executor = executor;
m_logger = logger;
m_buildArguments = args;
m_buildContext = buildContext;
}
/// <summary>
/// Executes a build depending on orchestrator / worker context
/// </summary>
/// <returns>The exit code returned by the worker process</returns>
public async Task<int> BuildAsync(bool isOrchestrator)
{
// Possibly extend context with additional info that can influence the build environment as needed
m_executor.PrepareBuildEnvironment(m_buildContext);
int returnCode;
if (isOrchestrator)
{
// The orchestrator creates the build info and publishes it to the build properties
var buildInfo = new BuildInfo { RelatedSessionId = Guid.NewGuid().ToString("D"), OrchestratorLocation = m_buildContext.AgentMachineName };
await m_vstsApi.PublishBuildInfo(m_buildContext, buildInfo);
returnCode = m_executor.ExecuteDistributedBuildAsOrchestrator(m_buildContext, buildInfo.RelatedSessionId, m_buildArguments);
}
else
{
// Get the build info from the orchestrator build
var buildInfo = await m_vstsApi.WaitForBuildInfo(m_buildContext);
returnCode = m_executor.ExecuteDistributedBuildAsWorker(m_buildContext, buildInfo, m_buildArguments);
}
LogExitCode(returnCode);
return returnCode;
}
private void LogExitCode(int returnCode)
{
if (returnCode != 0)
{
m_logger.Error(($"ExitCode: {returnCode}"));
}
else
{
m_logger.Info($"ExitCode: {returnCode}");
}
}
}
}

59
Public/Src/Tools/AdoBuildRunner/Program.cs
Просмотреть файл

@ -66,7 +66,7 @@ namespace BuildXL.AdoBuildRunner
logger.Info("Performing connectivity test");
executor = new PingExecutor(logger, api);
var buildManager = new BuildManager(api, executor, buildContext, args, logger);
return await buildManager.BuildAsync();
return await buildManager.BuildAsync(isOrchestrator: true);
}
else if (args[0] == "launchworkers")
{
@ -92,45 +92,46 @@ namespace BuildXL.AdoBuildRunner
executor = new BuildExecutor(logger);
bool isOrchestrator;
if (string.IsNullOrEmpty(role))
{
var buildContext = await api.GetBuildContextAsync("ParallelBuild");
var buildManager = new BuildManager(api, executor, buildContext, args, logger);
return await buildManager.BuildAsync();
// When the build role is not specified, we assume this build is being run with the parallel strategy
// where the role is inferred from the ordinal position in the phase: the first agent is the orchestrator
isOrchestrator = api.JobPositionInPhase == 1;
}
else
{
bool isWorker = string.Equals(role, "Worker", StringComparison.OrdinalIgnoreCase);
bool isOrchestrator = string.Equals(role, "Orchestrator", StringComparison.OrdinalIgnoreCase);
isOrchestrator = string.Equals(role, "Orchestrator", StringComparison.OrdinalIgnoreCase);
if (!isWorker && !isOrchestrator)
{
throw new CoordinationException($"{Constants.AdoBuildRunnerPipelineRole} must be 'Worker' or 'Orchestrator'");
}
// A build key has to be specified to disambiguate between multiple builds
// running as part of the same pipeline. This value is used to communicate
// the build information (orchestrator location, session id) to the workers.
var invocationKey = Environment.GetEnvironmentVariable(Constants.AdoBuildRunnerInvocationKey);
if (string.IsNullOrEmpty(invocationKey))
{
throw new CoordinationException($"The environment variable {Constants.AdoBuildRunnerInvocationKey} must be set (to a value that is unique within a particular pipeline run): " +
$"it is used to disambiguate between multiple builds running as part of the same pipeline (e.g.: debug, ship, test...) " +
$"and to communicate the build information to the worker pipeline");
}
var attemptNumber = Environment.GetEnvironmentVariable(Constants.JobAttemptVariableName) ?? "1";
if (int.TryParse(attemptNumber, out var jobAttempt) && jobAttempt > 1)
{
// The job was rerun. Let's change the invocation key to reflect that
// so we don't conflict with the first run.
invocationKey += $"__jobretry_{jobAttempt}";
}
var buildContext = await api.GetBuildContextAsync(invocationKey);
var buildManager = new WorkerPipelineBuildManager(api, executor, buildContext, args, logger);
return await buildManager.BuildAsync(isOrchestrator);
}
// A build key has to be specified to disambiguate between multiple builds
// running as part of the same pipeline. This value is used to communicate
// the build information (orchestrator location, session id) to the workers.
var invocationKey = Environment.GetEnvironmentVariable(Constants.AdoBuildRunnerInvocationKey);
if (string.IsNullOrEmpty(invocationKey))
{
throw new CoordinationException($"The environment variable {Constants.AdoBuildRunnerInvocationKey} must be set (to a value that is unique within a particular pipeline run): " +
$"it is used to disambiguate between multiple builds running as part of the same pipeline (e.g.: debug, ship, test...) " +
$"and to communicate the build information to the worker pipeline");
}
var attemptNumber = Environment.GetEnvironmentVariable(Constants.JobAttemptVariableName) ?? "1";
if (int.TryParse(attemptNumber, out var jobAttempt) && jobAttempt > 1)
{
// The job was rerun. Let's change the invocation key to reflect that
// so we don't conflict with the first run.
invocationKey += $"__jobretry_{jobAttempt}";
}
var buildContext = await api.GetBuildContextAsync(invocationKey);
var buildManager = new BuildManager(api, executor, buildContext, args, logger);
return await buildManager.BuildAsync(isOrchestrator);
}
}
catch (CoordinationException e)

18
Public/Src/Tools/AdoBuildRunner/Vsts/Api.cs
Просмотреть файл

@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Diagnostics.Contracts;
using System.Linq;
using System.Threading.Tasks;
using AdoBuildRunner.Vsts;
@ -291,9 +292,9 @@ namespace BuildXL.AdoBuildRunner.Vsts
}
// (2) One-to-one correspondence between a jobs in a worker pipeline, and an orchestrator
if (JobPositionInPhase != 1)
if (JobPositionInPhase != TotalJobsInPhase)
{
// Only do this once per 'parallel group', i.e., only for the first agent in a parallel strategy context
// Only do this once per 'parallel group', i.e., only for the last agent in a parallel strategy context
// (this means only one worker per distributed build). This is because there is no value that we can
// reliably use that is unique to a job but shared amongst the parallel agents of the same 'job'.
// But because parallel agents running the same job are exact replicas of each other (modulo 'JobPositionInPhase')
@ -425,15 +426,12 @@ namespace BuildXL.AdoBuildRunner.Vsts
public async Task<BuildInfo> WaitForBuildInfo(BuildContext buildContext)
{
var triggerInfo = await m_http.GetBuildTriggerInfoAsync();
if (triggerInfo == null)
if (triggerInfo == null
|| !triggerInfo.TryGetValue(Constants.TriggeringAdoBuildIdParameter, out string? triggeringBuildIdString)
|| !int.TryParse(triggeringBuildIdString, out int triggeringBuildId))
{
LogAndThrow("TriggerInfo is required to query the BuildInfo");
}
int triggeringBuildId = 0;
if (!triggerInfo.TryGetValue(Constants.TriggeringAdoBuildIdParameter, out string? triggeringBuildIdString) || !int.TryParse(triggeringBuildIdString, out triggeringBuildId))
{
LogAndThrow($"A worker build needs the value {Constants.TriggeringAdoBuildIdParameter} in the trigger info to be set to the orchestrator's ADO build id to connect to the build. Found this value for: {triggeringBuildIdString}");
m_logger.Info("Couldn't find trigger info for this build. Assuming it is being ran on the same pipeline as the orchestrator and using the current build id as the orchestrator's build id");
triggeringBuildId = int.Parse(BuildId);
}
var elapsedTime = 0;