From 5d3a2d4256b43055360f3eea666d13066339c3cf Mon Sep 17 00:00:00 2001 From: Marcelo Lynch Date: Fri, 8 Nov 2024 23:06:43 +0000 Subject: [PATCH] Merged PR 813911: Rings and overrides for the build tools installer Rings and overrides for the build tools installer --- Private/BuildToolsInstaller/.gitignore | 3 +- .../src/BuildToolsInstaller.cs | 31 +++- .../src/Config/BuildXLGlobalConfig.cs | 2 +- .../src/Config/BuildXLNugetInstallerConfig.cs | 1 - .../src/Config/DeploymentConfiguration.cs | 73 +++++++++ .../BuildXL/BuildXLNugetInstaller.cs | 52 +++---- .../src/Installers/IToolInstaller.cs | 7 +- Private/BuildToolsInstaller/src/Program.cs | 11 +- .../{AdoUtilities.cs => AdoService.cs} | 99 ++++++++---- .../src/Utiltiies/ConfigurationUtilities.cs | 65 ++++++++ .../src/Utiltiies/IAdoService.cs | 58 +++++++ .../{JsonDeserializer.cs => JsonUtilities.cs} | 18 ++- .../test/AdoUtilitiesTest.cs | 12 +- .../test/BuildXLInstallerTests.cs | 52 ++++--- .../test/DeploymentConfigurationTests.cs | 144 ++++++++++++++++++ .../test/MockAdoService.cs | 79 ++++++++++ .../BuildToolsInstaller/test/TestLogger.cs | 5 - 17 files changed, 600 insertions(+), 112 deletions(-) create mode 100644 Private/BuildToolsInstaller/src/Config/DeploymentConfiguration.cs rename Private/BuildToolsInstaller/src/Utiltiies/{AdoUtilities.cs => AdoService.cs} (53%) create mode 100644 Private/BuildToolsInstaller/src/Utiltiies/ConfigurationUtilities.cs create mode 100644 Private/BuildToolsInstaller/src/Utiltiies/IAdoService.cs rename Private/BuildToolsInstaller/src/Utiltiies/{JsonDeserializer.cs => JsonUtilities.cs} (83%) create mode 100644 Private/BuildToolsInstaller/test/DeploymentConfigurationTests.cs create mode 100644 Private/BuildToolsInstaller/test/MockAdoService.cs diff --git a/Private/BuildToolsInstaller/.gitignore b/Private/BuildToolsInstaller/.gitignore index b2288bb80..4aedd38e8 100644 --- a/Private/BuildToolsInstaller/.gitignore +++ b/Private/BuildToolsInstaller/.gitignore @@ -1,3 +1,4 @@ src/bin/* test/bin/* -*.nupkg \ No newline at end of file +*.nupkg +launchSettings.json \ No newline at end of file diff --git a/Private/BuildToolsInstaller/src/BuildToolsInstaller.cs b/Private/BuildToolsInstaller/src/BuildToolsInstaller.cs index 82be27c87..f738cd70a 100644 --- a/Private/BuildToolsInstaller/src/BuildToolsInstaller.cs +++ b/Private/BuildToolsInstaller/src/BuildToolsInstaller.cs @@ -1,12 +1,14 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using BuildToolsInstaller.Config; using BuildToolsInstaller.Logging; using BuildToolsInstaller.Utiltiies; +using NuGet.Common; namespace BuildToolsInstaller { - public record struct BuildToolsInstallerArgs(BuildTool Tool, string ToolsDirectory, string? ConfigFilePath, bool ForceInstallation); + public record struct BuildToolsInstallerArgs(BuildTool Tool, string? Ring, string ToolsDirectory, string? ConfigFilePath, bool ForceInstallation); /// /// Entrypoint for the installation logic @@ -21,19 +23,36 @@ namespace BuildToolsInstaller /// public static async Task Run(BuildToolsInstallerArgs arguments) { - // First, detect if we are running in an ADO build. - // This tool is meant to be run on a build, but is also run at image-creation - ILogger logger = AdoUtilities.IsAdoBuild ? new AdoConsoleLogger() : new ConsoleLogger(); + // This tool is primarily run on ADO, but could be also run locally, so we switch on IsEnabled when + // we want to do ADO-specific operations. + var adoService = AdoService.Instance; + ILogger logger = adoService.IsEnabled ? new AdoConsoleLogger() : new ConsoleLogger(); + + const string ConfigurationWellKnownUri = "https://bxlscripts.z20.web.core.windows.net/config/DeploymentConfig_V0.json"; + var deploymentConfiguration = await JsonUtilities.DeserializeFromHttpAsync(new Uri(ConfigurationWellKnownUri), logger, default); + if (deploymentConfiguration == null) + { + // Error should have been logged. + return FailureExitCode; + } IToolInstaller installer = arguments.Tool switch { - BuildTool.BuildXL => new BuildXLNugetInstaller(new NugetDownloader(), logger), + BuildTool.BuildXL => new BuildXLNugetInstaller(new NugetDownloader(), adoService, logger), // Shouldn't happen - the argument comes from a TryParse that should have failed earlier _ => throw new NotImplementedException($"No tool installer for tool {arguments.Tool}"), }; - return await installer.InstallAsync(arguments) ? SuccessExitCode : FailureExitCode; + var selectedRing = arguments.Ring ?? installer.DefaultRing; + var resolvedVersion = ConfigurationUtilities.ResolveVersion(deploymentConfiguration, selectedRing, arguments.Tool, adoService, logger); + if (resolvedVersion == null) + { + logger.Error("Failed to resolve version to install. Installation has failed."); + return FailureExitCode; + } + + return await installer.InstallAsync(resolvedVersion, arguments) ? SuccessExitCode : FailureExitCode; } } } diff --git a/Private/BuildToolsInstaller/src/Config/BuildXLGlobalConfig.cs b/Private/BuildToolsInstaller/src/Config/BuildXLGlobalConfig.cs index f922b7219..ff7e812ec 100644 --- a/Private/BuildToolsInstaller/src/Config/BuildXLGlobalConfig.cs +++ b/Private/BuildToolsInstaller/src/Config/BuildXLGlobalConfig.cs @@ -14,6 +14,6 @@ namespace BuildToolsInstaller.Config /// /// Latest release version number /// - public required string Release { get; set; } + public required string Release { get; init; } } } diff --git a/Private/BuildToolsInstaller/src/Config/BuildXLNugetInstallerConfig.cs b/Private/BuildToolsInstaller/src/Config/BuildXLNugetInstallerConfig.cs index 751c51448..b91fbb001 100644 --- a/Private/BuildToolsInstaller/src/Config/BuildXLNugetInstallerConfig.cs +++ b/Private/BuildToolsInstaller/src/Config/BuildXLNugetInstallerConfig.cs @@ -23,7 +23,6 @@ namespace BuildToolsInstaller.Config { /// /// A specific version to install - /// TODO: Support 'Latest' here /// public string? Version { get; set; } diff --git a/Private/BuildToolsInstaller/src/Config/DeploymentConfiguration.cs b/Private/BuildToolsInstaller/src/Config/DeploymentConfiguration.cs new file mode 100644 index 000000000..39d2a6ee1 --- /dev/null +++ b/Private/BuildToolsInstaller/src/Config/DeploymentConfiguration.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace BuildToolsInstaller.Config +{ + /// + /// Deployment details for a single tool + /// + /// + /// For now this just encapsulates a version, + /// but leaving it as an object for forwards extensibility + /// + public class ToolDeployment + { + public required string Version { get; set; } + } + + public class RingDefinition + { + /// + /// The identifier for the ring + /// + public required string Name { get; init; } + + /// + /// An optional description + /// + public string? Description { get; init; } + + /// + /// Tools available for this ring + /// + public required IReadOnlyDictionary Tools { get; init; } + } + + /// + /// An override to the default version of a tool that would be resolved for a build based on its selected ring + /// + public class DeploymentOverride + { + /// + /// Optional comment describing the override + /// + public string? Comment { get; init; } + + /// + /// The exception is applied to builds running under the repository with this name + /// + public required string Repository { get; init; } + + /// + /// If this is defined, the exception applies only to the specified pipelines + /// + public IReadOnlyList? PipelineIds { get; init; } + + /// + /// The overrides for this exception + /// + public required IReadOnlyDictionary Tools { get; init; } + } + + /// + /// The main configuration object + /// + public class DeploymentConfiguration + { + /// + public required IReadOnlyList Rings { get; init; } + + /// + public IReadOnlyList? Overrides { get; init; } + } +} diff --git a/Private/BuildToolsInstaller/src/Installers/BuildXL/BuildXLNugetInstaller.cs b/Private/BuildToolsInstaller/src/Installers/BuildXL/BuildXLNugetInstaller.cs index ec84bda86..8602d392e 100644 --- a/Private/BuildToolsInstaller/src/Installers/BuildXL/BuildXLNugetInstaller.cs +++ b/Private/BuildToolsInstaller/src/Installers/BuildXL/BuildXLNugetInstaller.cs @@ -19,6 +19,10 @@ namespace BuildToolsInstaller /// public class BuildXLNugetInstaller : IToolInstaller { + // Default ring for BuildXL installation + public string DefaultRing => "GeneralPublic"; + + private readonly IAdoService m_adoService; private readonly INugetDownloader m_downloader; private readonly ILogger m_logger; private BuildXLNugetInstallerConfig? m_config; @@ -30,14 +34,15 @@ namespace BuildToolsInstaller // Use only after calling TryInitializeConfig() private BuildXLNugetInstallerConfig Config => m_config!; - public BuildXLNugetInstaller(INugetDownloader downloader, ILogger logger) + public BuildXLNugetInstaller(INugetDownloader downloader, IAdoService adoService, ILogger logger) { + m_adoService = adoService; m_downloader = downloader; m_logger = logger; } /// - public async Task InstallAsync(BuildToolsInstallerArgs args) + public async Task InstallAsync(string selectedVersion, BuildToolsInstallerArgs args) { if (!await TryInitializeConfigAsync(args)) { @@ -46,10 +51,11 @@ namespace BuildToolsInstaller try { - var version = await TryResolveVersionAsync(); + // Version override + var version = await TryResolveVersionAsync(selectedVersion); if (version == null) { - m_logger.Error("BuildLXNugetInstaller: failed to resolve version to install."); + // Error should have been logged return false; } @@ -85,7 +91,7 @@ namespace BuildToolsInstaller return false; } - var feed = Config.FeedOverride ?? InferSourceRepository(); + var feed = Config.FeedOverride ?? InferSourceRepository(m_adoService); var repository = CreateSourceRepository(feed); if (await m_downloader.TryDownloadNugetToDiskAsync(repository, PackageName, nugetVersion, downloadLocation, m_logger)) @@ -120,14 +126,14 @@ namespace BuildToolsInstaller /// /// Construct the implicit source repository for installers, a well-known feed that should be installed in the organization /// - private static string InferSourceRepository() + private static string InferSourceRepository(IAdoService adoService) { - if (!AdoUtilities.IsAdoBuild) + if (!adoService.IsEnabled) { throw new InvalidOperationException("Automatic source repository inference is only supported when running on an ADO Build"); } - if (!AdoUtilities.TryGetOrganizationName(out var adoOrganizationName)) + if (!adoService.TryGetOrganizationName(out var adoOrganizationName)) { throw new InvalidOperationException("Could not retrieve organization name"); } @@ -152,7 +158,10 @@ namespace BuildToolsInstaller private void SetLocationVariable(string engineLocation) { - AdoUtilities.SetVariable("ONEES_BUILDXL_LOCATION", engineLocation, isReadOnly: true); + if (m_adoService.IsEnabled) + { + m_adoService.SetVariable("ONEES_BUILDXL_LOCATION", engineLocation, isReadOnly: true); + } } private async Task TryInitializeConfigAsync(BuildToolsInstallerArgs args) @@ -163,14 +172,14 @@ namespace BuildToolsInstaller return true; } - m_config = await JsonDeserializer.DeserializeAsync(args.ConfigFilePath, m_logger, CancellationToken.None); + m_config = await JsonUtilities.DeserializeAsync(args.ConfigFilePath, m_logger, serializerOptions: new () { PropertyNameCaseInsensitive = true }); if (m_config == null) { m_logger.Error("Could not parse the BuildXL installer configuration. Installation will fail."); return false; } - if (m_config.DistributedRole != null && !AdoUtilities.IsAdoBuild) + if (m_config.DistributedRole != null && !m_adoService.IsEnabled) { m_logger.Error("Distributed mode can only be enabld in ADO Builds. Installation will fail."); return false; @@ -187,7 +196,7 @@ namespace BuildToolsInstaller return Environment.GetEnvironmentVariable("BUILDTOOLSDOWNLOADER_NUGET_PAT") ?? Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN") ?? ""; } - private async Task TryResolveVersionAsync() + private async Task TryResolveVersionAsync(string selectedVersion) { var resolvedVersionProperty = "BuildXLResolvedVersion_" + (Config.InvocationKey ?? "default"); string? resolvedVersion = null; @@ -216,25 +225,16 @@ namespace BuildToolsInstaller } else { - // Orchestrator (or default) mode -> resolve version from global configuration an publish it - const string ConfigurationWellKnownUri = "https://bxlscripts.z20.web.core.windows.net/config/buildxl/BuildXLConfig_V0.json"; - var jsonUri = new Uri(Config.Internal_GlobalConfigOverride ?? ConfigurationWellKnownUri); - var config = await JsonDeserializer.DeserializeFromHttpAsync(jsonUri, m_logger, default); - if (config == null) - { - // Error should have been logged. - return null; - } - - resolvedVersion = config.Release; + // Orchestrator (or default) mode + resolvedVersion = selectedVersion; } - if (Config.DistributedRole== DistributedRole.Orchestrator) + if (Config.DistributedRole == DistributedRole.Orchestrator) { // We resolved a version - we should push it to the properties for the workers to consume. // If the version is null it means we encountered some error above, so push the empty string // (the workers must be signalled that there was an error somehow). - await AdoUtilities.SetBuildPropertyAsync(resolvedVersionProperty, resolvedVersion ?? string.Empty); + await m_adoService.SetBuildPropertyAsync(resolvedVersionProperty, resolvedVersion ?? string.Empty); } return resolvedVersion; @@ -247,7 +247,7 @@ namespace BuildToolsInstaller { token.ThrowIfCancellationRequested(); - var maybeProperty = await AdoUtilities.GetBuildPropertyAsync(propertyKey); + var maybeProperty = await m_adoService.GetBuildPropertyAsync(propertyKey); if (maybeProperty != null) { // Orchestrator pushes an empty string on error diff --git a/Private/BuildToolsInstaller/src/Installers/IToolInstaller.cs b/Private/BuildToolsInstaller/src/Installers/IToolInstaller.cs index 23967d3c4..73d394e0c 100644 --- a/Private/BuildToolsInstaller/src/Installers/IToolInstaller.cs +++ b/Private/BuildToolsInstaller/src/Installers/IToolInstaller.cs @@ -11,6 +11,11 @@ namespace BuildToolsInstaller /// /// Install to the given directory /// - public Task InstallAsync(BuildToolsInstallerArgs args); + public Task InstallAsync(string selectedVersion, BuildToolsInstallerArgs args); + + /// + /// The name of the default ring for this tool + /// + public string DefaultRing { get; } } } diff --git a/Private/BuildToolsInstaller/src/Program.cs b/Private/BuildToolsInstaller/src/Program.cs index eaa4a8c92..f2bff1b42 100644 --- a/Private/BuildToolsInstaller/src/Program.cs +++ b/Private/BuildToolsInstaller/src/Program.cs @@ -23,6 +23,10 @@ namespace BuildToolsInstaller description: "The tool to install.") { IsRequired = true }; + var ringOption = new Option( + name: "--ring", + description: "Selects a deployment ring for the tool"); + var toolsDirectoryOption = new Option( name: "--toolsDirectory", description: "The location where packages should be downloaded. Defaults to AGENT_TOOLSDIRECTORY if defined, or the working directory if not"); @@ -54,23 +58,26 @@ namespace BuildToolsInstaller var rootCommand = new RootCommand("Build tools installer"); rootCommand.AddOption(toolOption); + rootCommand.AddOption(ringOption); rootCommand.AddOption(toolsDirectoryOption); rootCommand.AddOption(configOption); rootCommand.AddOption(forceOption); int returnCode = ProgramNotRunExitCode; // Make the compiler happy, we should assign every time - rootCommand.SetHandler(async (tool, toolsDirectory, configFile, forceInstallation) => + rootCommand.SetHandler(async (tool, ring, toolsDirectory, configFile, forceInstallation) => { - toolsDirectory ??= AdoUtilities.ToolsDirectory ?? "."; + toolsDirectory ??= AdoService.Instance.IsEnabled ? AdoService.Instance.ToolsDirectory : "."; returnCode = await BuildToolsInstaller.Run(new BuildToolsInstallerArgs() { Tool = tool, + Ring = ring, ToolsDirectory = toolsDirectory, ConfigFilePath = configFile, ForceInstallation = forceInstallation }); }, toolOption, + ringOption, toolsDirectoryOption, configOption, forceOption diff --git a/Private/BuildToolsInstaller/src/Utiltiies/AdoUtilities.cs b/Private/BuildToolsInstaller/src/Utiltiies/AdoService.cs similarity index 53% rename from Private/BuildToolsInstaller/src/Utiltiies/AdoUtilities.cs rename to Private/BuildToolsInstaller/src/Utiltiies/AdoService.cs index c6e4a9b50..01e448535 100644 --- a/Private/BuildToolsInstaller/src/Utiltiies/AdoUtilities.cs +++ b/Private/BuildToolsInstaller/src/Utiltiies/AdoService.cs @@ -1,56 +1,89 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Text.RegularExpressions; using Microsoft.TeamFoundation.Build.WebApi; using Microsoft.VisualStudio.Services.Common; using Microsoft.VisualStudio.Services.WebApi; -using NuGet.Protocol.Plugins; namespace BuildToolsInstaller.Utiltiies { - internal sealed partial class AdoUtilities + internal sealed partial class AdoService : IAdoService { + // Make this class a singleton + private AdoService() { } + + public static AdoService Instance { get; } = s_instance ??= new(); + // Keep as lazily initialized for the sake of testing outside of ADO (where we need to modify the environment first) + private static AdoService? s_instance; + /// /// True if the process is running in an ADO build. /// The other methods and properties in this class are meaningful if this is true. /// - public static bool IsAdoBuild => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD")); + public bool IsEnabled => m_isAdoBuild; + private readonly bool m_isAdoBuild = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TF_BUILD")); [GeneratedRegex(@"^(?:https://(?[a-zA-Z0-9-]+)\.(?:vsrm\.)?visualstudio\.com/|https://(?:vsrm\.)?dev\.azure\.com/(?[a-zA-Z0-9-]+)/)$", RegexOptions.CultureInvariant)] private static partial Regex CollectionUriRegex(); - /// - public static string CollectionUri => Environment.GetEnvironmentVariable("SYSTEM_COLLECTIONURI")!; - /// - public static string ToolsDirectory => Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY")!; - - /// - public static string AccessToken => Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN")!; - - /// - private static string ServerUri => Environment.GetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONSERVERURI")!; - - /// - private static string ProjectId => Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECTID")!; - - /// - public static string BuildId => Environment.GetEnvironmentVariable("BUILD_BUILDID")!; - - private static BuildHttpClient BuildClient => s_httpClient ??= new BuildHttpClient(new Uri(ServerUri), new VssBasicCredential(string.Empty, AccessToken)); - private static BuildHttpClient? s_httpClient; - - public static async Task GetBuildPropertyAsync(string key) + private void EnsureAdo() { + if (!IsEnabled) + { + throw new InvalidOperationException($"This operation in {nameof(AdoService)} is only available in an ADO build"); + } + } + + private T EnsuringAdo(T? ret) + { + EnsureAdo(); + return ret!; + } + + #region Predefined variables - see https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables + /// + public string CollectionUri => EnsuringAdo(Environment.GetEnvironmentVariable("SYSTEM_COLLECTIONURI")); + + /// + public string ToolsDirectory => EnsuringAdo(Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY")); + + /// + public string AccessToken => EnsuringAdo(Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN")); + + /// + private string ServerUri => EnsuringAdo(Environment.GetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONSERVERURI")); + + /// + private string ProjectId => EnsuringAdo(Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECTID")); + + /// + public string BuildId => EnsuringAdo(Environment.GetEnvironmentVariable("BUILD_BUILDID")); + + /// + public string RepositoryName => Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")!; + + /// + public int PipelineId => int.Parse(Environment.GetEnvironmentVariable("SYSTEM_DEFINITIONID")!); + #endregion + + private BuildHttpClient BuildClient => m_httpClient ??= new BuildHttpClient(new Uri(ServerUri), new VssBasicCredential(string.Empty, AccessToken)); + private BuildHttpClient? m_httpClient; + + /// + public async Task GetBuildPropertyAsync(string key) + { + EnsureAdo(); var props = await IdempotentWithRetry(() => BuildClient.GetBuildPropertiesAsync(ProjectId, int.Parse(BuildId))); return props.ContainsKey(key) ? props.GetValue(key, string.Empty) : null; } - public static async Task SetBuildPropertyAsync(string key, string value) + /// + public async Task SetBuildPropertyAsync(string key, string value) { - + EnsureAdo(); // UpdateBuildProperties is ultimately an HTTP PATCH: the new properties specified will be added to the existing ones // in an atomic fashion. So we don't have to worry about multiple builds concurrently calling UpdateBuildPropertiesAsync // as long as the keys don't clash. @@ -82,8 +115,9 @@ namespace BuildToolsInstaller.Utiltiies /// /// Get the organization name from environment data in the agent /// - public static bool TryGetOrganizationName([NotNullWhen(true)] out string? organizationName) + public bool TryGetOrganizationName([NotNullWhen(true)] out string? organizationName) { + EnsureAdo(); organizationName = null; string collectionUri = CollectionUri; if (collectionUri == null) @@ -105,12 +139,11 @@ namespace BuildToolsInstaller.Utiltiies } } - /// - /// Set a variable that will be visible by subsequent tasks in the running job - /// - public static void SetVariable(string variableName, string value, bool isReadOnly = true) + /// + public void SetVariable(string variableName, string value, bool isReadOnly = true) { - if (!IsAdoBuild) + EnsureAdo(); + if (!IsEnabled) { return; } diff --git a/Private/BuildToolsInstaller/src/Utiltiies/ConfigurationUtilities.cs b/Private/BuildToolsInstaller/src/Utiltiies/ConfigurationUtilities.cs new file mode 100644 index 000000000..5fbfadc30 --- /dev/null +++ b/Private/BuildToolsInstaller/src/Utiltiies/ConfigurationUtilities.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; +using BuildToolsInstaller.Config; + +namespace BuildToolsInstaller.Utiltiies +{ + internal class ConfigurationUtilities + { + public static string? ResolveVersion(DeploymentConfiguration deploymentConfiguration, string ring, BuildTool tool, IAdoService adoService, ILogger logger) + { + // 1. Check overrides + if (TryGetFromOverride(deploymentConfiguration, tool, adoService, out var resolvedVersion, logger)) + { + return resolvedVersion; + } + + // 2. Resolve from ring + var selectedRing = deploymentConfiguration.Rings.FirstOrDefault(r => r.Name == ring); + if (selectedRing == null) + { + logger.Error($"Could not find configuration for ring {ring}. Available rings are: [{string.Join(", ", deploymentConfiguration.Rings.Select(r => r.Name))}]"); + return null; + } + + if(!selectedRing.Tools.TryGetValue(tool, out var resolved)) + { + logger.Error($"Could not find configuration for tool {tool} in ring {ring}."); + return null; + } + + return resolved.Version; + } + + private static bool TryGetFromOverride(DeploymentConfiguration deploymentConfiguration, BuildTool tool, IAdoService adoService, [NotNullWhen(true)] out string? resolvedVersion, ILogger logger) + { + resolvedVersion = null; + if (!adoService.IsEnabled || deploymentConfiguration.Overrides is null || deploymentConfiguration.Overrides.Count == 0) + { + // Overrides can only be applied when running an ADO build + return false; + } + + var repository = adoService.RepositoryName; + var pipelineId = adoService.PipelineId; + + foreach (var exception in deploymentConfiguration.Overrides) + { + if (string.Equals(exception.Repository, repository, StringComparison.OrdinalIgnoreCase) + && (exception.PipelineIds == null || exception.PipelineIds.Contains(pipelineId)) + && exception.Tools.TryGetValue(tool, out var toolDeployment)) + { + resolvedVersion = toolDeployment.Version; + var details = string.IsNullOrEmpty(exception.Comment) ? string.Empty : $" Details: {exception.Comment}"; + logger.Info($"Selecting version {resolvedVersion} for tool {tool} from a global configuration override.{details}"); + return true; + } + } + + // No matches + return false; + } + } +} diff --git a/Private/BuildToolsInstaller/src/Utiltiies/IAdoService.cs b/Private/BuildToolsInstaller/src/Utiltiies/IAdoService.cs new file mode 100644 index 000000000..dd8413e3f --- /dev/null +++ b/Private/BuildToolsInstaller/src/Utiltiies/IAdoService.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics.CodeAnalysis; + +namespace BuildToolsInstaller.Utiltiies +{ + /// + /// Component that interacts with Azure DevOps, using the REST API or the environment + /// Abstracted as an interface for ease of testing + /// + public interface IAdoService + { + /// + /// Whether ADO interaction is enabled + /// This is false when not running on an agent, and other methods may throw in this case + /// + public bool IsEnabled { get; } + + /// + public string CollectionUri { get; } + + /// + public string ToolsDirectory { get; } + + /// + public string AccessToken { get; } + + /// + public string BuildId { get; } + + /// + public string RepositoryName { get; } + + /// + public int PipelineId { get; } + + /// + /// Sets a build property with the specified value using the ADO REST API + /// + public Task SetBuildPropertyAsync(string key, string value); + + /// + /// Gets the value of a build property using the REST API, or null if such property is not defined + /// + public Task GetBuildPropertyAsync(string key); + + /// + /// Get the organization name from environment data in the agent + /// + public bool TryGetOrganizationName([NotNullWhen(true)] out string? organizationName); + + /// + /// Set a variable that will be visible by subsequent tasks in the running job + /// + public void SetVariable(string variableName, string value, bool isReadOnly = true); + } +} diff --git a/Private/BuildToolsInstaller/src/Utiltiies/JsonDeserializer.cs b/Private/BuildToolsInstaller/src/Utiltiies/JsonUtilities.cs similarity index 83% rename from Private/BuildToolsInstaller/src/Utiltiies/JsonDeserializer.cs rename to Private/BuildToolsInstaller/src/Utiltiies/JsonUtilities.cs index 24f7ab63c..68b40fe26 100644 --- a/Private/BuildToolsInstaller/src/Utiltiies/JsonDeserializer.cs +++ b/Private/BuildToolsInstaller/src/Utiltiies/JsonUtilities.cs @@ -10,16 +10,20 @@ namespace BuildToolsInstaller.Utiltiies /// /// Deserializing utilities /// - public static class JsonDeserializer + public static class JsonUtilities { private static readonly HttpClient s_httpClient = new HttpClient(); private const int MaxRetries = 3; private static readonly TimeSpan s_delayBetweenRetries = TimeSpan.FromSeconds(2); + internal static readonly JsonSerializerOptions DefaultSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; /// /// Deserialize from a file stored in an Azure Storage blob, logging an error and returning null if the operation fails /// - public static async Task DeserializeFromBlobAsync(Uri blobUri, ILogger logger, CancellationToken token) + public static async Task DeserializeFromBlobAsync(Uri blobUri, ILogger logger, JsonSerializerOptions? serializerOptions = null, CancellationToken token = default) { // The storage account should have been configured with anonymous read access to the config blob, // so no need to provide any credentials. @@ -39,7 +43,7 @@ namespace BuildToolsInstaller.Utiltiies return default; } - return await DeserializeAsync(downloadPath, logger, token); + return await DeserializeAsync(downloadPath, logger, serializerOptions, token); } finally { @@ -57,7 +61,7 @@ namespace BuildToolsInstaller.Utiltiies /// /// Deserializes a JSON file pointed by . The request is retried upon HTTP failures. /// - public static async Task DeserializeFromHttpAsync(Uri uri, ILogger logger, CancellationToken token) + public static async Task DeserializeFromHttpAsync(Uri uri, ILogger logger, JsonSerializerOptions? serializerOptions = null, CancellationToken token = default) { int retryCount = 0; @@ -70,7 +74,7 @@ namespace BuildToolsInstaller.Utiltiies response.EnsureSuccessStatusCode(); await using (Stream responseStream = await response.Content.ReadAsStreamAsync(token)) { - T? result = await JsonSerializer.DeserializeAsync(responseStream, new JsonSerializerOptions { PropertyNameCaseInsensitive = true }, token); + T? result = await JsonSerializer.DeserializeAsync(responseStream, serializerOptions ?? DefaultSerializerOptions, token); logger.Info($"Successfully deserialized JSON from {uri}."); return result; } @@ -102,13 +106,13 @@ namespace BuildToolsInstaller.Utiltiies /// /// Deserialize JSON from a file, logging an error and returning null if the operation fails /// - public static async Task DeserializeAsync(string filePath, ILogger logger, CancellationToken token) + public static async Task DeserializeAsync(string filePath, ILogger logger, JsonSerializerOptions? serializerOptions = null, CancellationToken token = default) { try { using (FileStream openStream = File.OpenRead(filePath)) { - return await JsonSerializer.DeserializeAsync(openStream, cancellationToken: token); + return await JsonSerializer.DeserializeAsync(openStream, serializerOptions ?? DefaultSerializerOptions, cancellationToken: token); } } catch (Exception e) diff --git a/Private/BuildToolsInstaller/test/AdoUtilitiesTest.cs b/Private/BuildToolsInstaller/test/AdoUtilitiesTest.cs index 68177aa19..23b1abdf8 100644 --- a/Private/BuildToolsInstaller/test/AdoUtilitiesTest.cs +++ b/Private/BuildToolsInstaller/test/AdoUtilitiesTest.cs @@ -1,12 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System; -using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BuildToolsInstaller.Utiltiies; using Xunit; @@ -26,9 +21,10 @@ namespace BuildToolsInstaller.Tests modifyEnvironment.Set("SYSTEM_COLLECTIONURI", "https://mseng.visualstudio.com/"); modifyEnvironment.Set("TF_BUILD", "True"); - Assert.True(AdoUtilities.IsAdoBuild); - Assert.Equal(toolsDirectory, AdoUtilities.ToolsDirectory); - Assert.True(AdoUtilities.TryGetOrganizationName(out var orgName)); + var adoService = AdoService.Instance; + Assert.True(adoService.IsEnabled); + Assert.Equal(toolsDirectory, adoService.ToolsDirectory); + Assert.True(adoService.TryGetOrganizationName(out var orgName)); Assert.Equal("mseng", orgName); } } diff --git a/Private/BuildToolsInstaller/test/BuildXLInstallerTests.cs b/Private/BuildToolsInstaller/test/BuildXLInstallerTests.cs index 974134206..b9bfe61bf 100644 --- a/Private/BuildToolsInstaller/test/BuildXLInstallerTests.cs +++ b/Private/BuildToolsInstaller/test/BuildXLInstallerTests.cs @@ -1,26 +1,24 @@ using System; -using System.ComponentModel.DataAnnotations; using System.IO; -using System.Linq; using System.Threading.Tasks; -using BuildToolsInstaller.Utiltiies; using Xunit; namespace BuildToolsInstaller.Tests { public class BuildXLInstallerTests { + const string DefaultRing = "GeneralPublic"; + const string DefaultVersion = "0.1.0-20252610.1"; + [Fact] public async Task AdoEnvironmentPickedUpByDefault() { - // We may modify the environment during the test, this will restore it on dispose - using var modifyEnvironment = new TemporaryTestEnvironment(); - - // If we're not in ADO, this will simulate that we are + // If we're not in ADO, this will simulate that we are var toolsDirectory = Path.Combine(Path.GetTempPath(), "Test"); - modifyEnvironment.Set("AGENT_TOOLSDIRECTORY", toolsDirectory); - modifyEnvironment.Set("SYSTEM_COLLECTIONURI", "https://mseng.visualstudio.com/"); - modifyEnvironment.Set("TF_BUILD", "True"); + var mockAdoService = new MockAdoService() + { + ToolsDirectory = toolsDirectory + }; var configTempPath = Path.GetTempFileName(); File.WriteAllText(configTempPath, """ @@ -33,16 +31,16 @@ namespace BuildToolsInstaller.Tests var mockDownloader = new MockNugetDownloader(); var log = new TestLogger(); - var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, AdoUtilities.ToolsDirectory, configTempPath, false); - var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, log); - var result = await buildXLInstaller.InstallAsync(args); + var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, "GeneralPublic", mockAdoService.ToolsDirectory, configTempPath, false); + var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, mockAdoService, log); + var result = await buildXLInstaller.InstallAsync(DefaultVersion, args); Assert.True(result, log.FullLog); Assert.Single(mockDownloader.Downloads); var download = mockDownloader.Downloads[0]; // Default feed is within the org - Assert.StartsWith("https://pkgs.dev.azure.com/mseng/_packaging", download.Repository); + Assert.StartsWith($"https://pkgs.dev.azure.com/{MockAdoService.OrgName}/_packaging", download.Repository); Assert.Equal("0.1.0-20241026.1", download.Version); } finally @@ -59,6 +57,12 @@ namespace BuildToolsInstaller.Tests [Fact] public async Task InstallWithCustomConfig() { + var downloadDirectory = Path.Join(Path.GetTempPath(), $"Test_{nameof(InstallWithCustomConfig)}"); + if (Directory.Exists(downloadDirectory)) + { + Directory.Delete(downloadDirectory, true); + } + var configTempPath = Path.GetTempFileName(); File.WriteAllText(configTempPath, """ { @@ -71,10 +75,15 @@ namespace BuildToolsInstaller.Tests var mockDownloader = new MockNugetDownloader(); var log = new TestLogger(); - var downloadDirectory = Path.GetTempPath(); - var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, downloadDirectory, configTempPath, false); - var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, log); - var result = await buildXLInstaller.InstallAsync(args); + var mockAdoService = new MockAdoService() + { + ToolsDirectory = downloadDirectory + }; + + var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, DefaultRing, downloadDirectory, configTempPath, false); + + var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, mockAdoService, log); + var result = await buildXLInstaller.InstallAsync(DefaultVersion, args); Assert.True(result, log.FullLog); Assert.Single(mockDownloader.Downloads); @@ -112,9 +121,10 @@ namespace BuildToolsInstaller.Tests var mockDownloader = new MockNugetDownloader(); var log = new TestLogger(); - var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, toolsDirectory, configTempPath, force); - var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, log); - var result = await buildXLInstaller.InstallAsync(args); + var args = new BuildToolsInstallerArgs(BuildTool.BuildXL, DefaultRing, toolsDirectory, configTempPath, force); + var mockAdoService = new MockAdoService() { ToolsDirectory = toolsDirectory }; + var buildXLInstaller = new BuildXLNugetInstaller(mockDownloader, mockAdoService, log); + var result = await buildXLInstaller.InstallAsync(DefaultVersion, args); Assert.True(result, log.FullLog); if (force) diff --git a/Private/BuildToolsInstaller/test/DeploymentConfigurationTests.cs b/Private/BuildToolsInstaller/test/DeploymentConfigurationTests.cs new file mode 100644 index 000000000..a9e2f1af6 --- /dev/null +++ b/Private/BuildToolsInstaller/test/DeploymentConfigurationTests.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Text.Json; +using BuildToolsInstaller.Config; +using BuildToolsInstaller.Utiltiies; +using Xunit; + +namespace BuildToolsInstaller.Tests +{ + public class DeploymentConfigurationTests + { + private const string TestConfiguration = @" +{ + ""rings"": [ + { + ""name"": ""Dogfood"", + ""description"": ""Dogfood ring"", + ""tools"": { + ""BuildXL"": { ""version"": ""0.1.0-20250101.1"" } + } + }, + { + ""name"": ""GeneralPublic"", + ""tools"": { + ""BuildXL"": { ""version"": ""0.1.0-20241025.4"" } + } + } + ], + + ""overrides"": + [ + { + ""comment"": ""description of the pin"", + ""repository"": ""1JS"", + ""tools"": { + ""BuildXL"": { ""version"": ""0.1.0-20240801.1"" } + } + }, + { + ""repository"": ""BuildXL.Internal"", + ""pipelineIds"": [10101, 1010], + ""tools"": { + ""BuildXL"": { ""version"": ""0.1.0-20240801.1"" } + } + } + ] +} +"; + + [Fact] + public void DeserializationTest() + { + var deserialized = JsonSerializer.Deserialize(TestConfiguration, JsonUtilities.DefaultSerializerOptions); + Assert.NotNull(deserialized); + Assert.NotNull(deserialized.Overrides); + Assert.NotNull(deserialized.Rings); + Assert.Equal(2, deserialized.Overrides.Count); + Assert.Equal(2, deserialized.Rings.Count); + Assert.Equal("Dogfood", deserialized.Rings[0].Name); + Assert.Equal("Dogfood ring", deserialized.Rings[0].Description); + Assert.Contains(BuildTool.BuildXL, deserialized.Rings[0].Tools.Keys); + Assert.Equal("0.1.0-20250101.1", deserialized.Rings[0].Tools[BuildTool.BuildXL].Version); + Assert.Equal("description of the pin", deserialized.Overrides[0].Comment); + Assert.Equal("1JS", deserialized.Overrides[0].Repository); + Assert.Contains(BuildTool.BuildXL, deserialized.Overrides[0].Tools.Keys); + Assert.Equal("0.1.0-20240801.1", deserialized.Overrides[0].Tools[BuildTool.BuildXL].Version); + Assert.NotNull(deserialized.Overrides[1].PipelineIds); + Assert.Equal(2, deserialized.Overrides[1].PipelineIds!.Count); + } + + [Fact] + public void ResolutionByRingTest() + { + var config = new DeploymentConfiguration() + { + Rings = [ + new RingDefinition() { + Name = "A", + Tools = new ReadOnlyDictionary(new Dictionary () { + { BuildTool.BuildXL, new ToolDeployment() { Version = "VersionA" } } + }) + }, + new RingDefinition() { + Name = "B", + Tools = new ReadOnlyDictionary(new Dictionary () { + { BuildTool.BuildXL, new ToolDeployment() { Version = "VersionB" } } + }) + } + ] + }; + + var mockAdoService = new MockAdoService() { ToolsDirectory = Path.GetTempPath() }; + Assert.Equal("VersionA", ConfigurationUtilities.ResolveVersion(config, "A", BuildTool.BuildXL, mockAdoService, new TestLogger())); + Assert.Equal("VersionB", ConfigurationUtilities.ResolveVersion(config, "B", BuildTool.BuildXL, mockAdoService, new TestLogger())); + Assert.Null(ConfigurationUtilities.ResolveVersion(config, "C", BuildTool.BuildXL, mockAdoService, new TestLogger())); + } + + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public void Exceptions(bool pinByRepo, bool pinByPipeline) + { + var pinnedRepo = "PinnedRepo"; + var pinnedPipeline = 1001; + + var config = new DeploymentConfiguration() + { + Rings = [ + new RingDefinition() { + Name = "A", + Tools = new ReadOnlyDictionary(new Dictionary () { + { BuildTool.BuildXL, new ToolDeployment() { Version = "VersionA" } } + }) + } + ], + Overrides = [ + new DeploymentOverride() { + Repository = pinByRepo ? pinnedRepo : "not_ " + pinnedRepo, + PipelineIds = pinByPipeline ? [ pinnedPipeline ] : null, + Tools = new ReadOnlyDictionary(new Dictionary () { + { BuildTool.BuildXL, new ToolDeployment() { Version = "PinnedVersion" } } + }) + } + ] + }; + + var mockAdoService = new MockAdoService() + { + ToolsDirectory = Path.GetTempPath(), + RepositoryName = pinnedRepo, + PipelineId = pinnedPipeline + }; + + var expected = pinByRepo ? "PinnedVersion" : "VersionA"; + Assert.Equal(expected, ConfigurationUtilities.ResolveVersion(config, "A", BuildTool.BuildXL, mockAdoService, new TestLogger())); + } + } +} diff --git a/Private/BuildToolsInstaller/test/MockAdoService.cs b/Private/BuildToolsInstaller/test/MockAdoService.cs new file mode 100644 index 000000000..1859cae3d --- /dev/null +++ b/Private/BuildToolsInstaller/test/MockAdoService.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using BuildToolsInstaller.Utiltiies; + +namespace BuildToolsInstaller.Tests +{ + internal class MockAdoService : IAdoService + { + public static readonly string OrgName = "testOrg"; + public bool IsEnabled => true; + + public string CollectionUri => $"https://dev.azure.com/{OrgName}/"; + + public required string ToolsDirectory { get; init; } + + public string AccessToken { get; init; } = ""; + + public string BuildId { get; init; } = "212121"; + + public string RepositoryName { get; init; } = "TestRepo"; + + public int PipelineId { get; init; } = 13012; + + public ConcurrentDictionary Properties = new(); + public Task GetBuildPropertyAsync(string key) + { + return Task.FromResult(Properties.TryGetValue(key, out var value) ? value : null); + } + + public Task SetBuildPropertyAsync(string key, string value) + { + Properties[key] = value; + return Task.CompletedTask; + } + + public void SetVariable(string variableName, string value, bool isReadOnly = true) + { + // No-Op for tests + } + + public bool TryGetOrganizationName([NotNullWhen(true)] out string? organizationName) + { + organizationName = OrgName; + return true; + } + } + + internal class DisabledMockAdoService : IAdoService + { + public bool IsEnabled => false; + + private Exception Error => new InvalidOperationException("AdoService is disabled"); + + public string CollectionUri => throw Error; + + public string ToolsDirectory => throw Error; + + public string AccessToken => throw Error; + + public string BuildId => throw Error; + + public string RepositoryName => throw Error; + + public int PipelineId => throw Error; + + public Task GetBuildPropertyAsync(string key) => throw Error; + + public Task SetBuildPropertyAsync(string key, string value) => throw Error; + + public void SetVariable(string variableName, string value, bool isReadOnly = true) => throw Error; + + public bool TryGetOrganizationName([NotNullWhen(true)] out string? organizationName) => throw Error; + } +} diff --git a/Private/BuildToolsInstaller/test/TestLogger.cs b/Private/BuildToolsInstaller/test/TestLogger.cs index 4e02b790d..6f1190f1c 100644 --- a/Private/BuildToolsInstaller/test/TestLogger.cs +++ b/Private/BuildToolsInstaller/test/TestLogger.cs @@ -2,13 +2,8 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Xunit.Sdk; namespace BuildToolsInstaller.Tests {