diff --git a/Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc b/Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc index a39b90dfa..64d0389c0 100644 --- a/Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc +++ b/Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc @@ -179,9 +179,9 @@ export function evaluate(args: Arguments): Result { ], allowedSurvivingChildProcessNames: [ "mspdbsrv.exe", - "VCTIP.exe", "conhost.exe", - ] + ], + unsafe: { childProcessesToBreakawayFromSandbox: [a`vctip.exe`] } }); let midlOutput = { diff --git a/Public/Sdk/Public/Managed/Tools/NuGet/empty.config b/Public/Sdk/Public/Managed/Tools/NuGet/empty.config index 0b66186ce..a342b8dfb 100644 --- a/Public/Sdk/Public/Managed/Tools/NuGet/empty.config +++ b/Public/Sdk/Public/Managed/Tools/NuGet/empty.config @@ -27,13 +27,16 @@ Each time a new nuget version is updated we must make sure that any configuratio - - - + + + + + + \ No newline at end of file diff --git a/Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc b/Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc index 32c029b2b..047ba8347 100644 --- a/Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc +++ b/Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc @@ -14,7 +14,8 @@ export const tool : Transformer.ToolDefinition = { exe: Nuget.Contents.all.getFile(r`tools/NuGet.exe`), description: "NuGet pack", untrackedDirectoryScopes: [ - d`${Context.getMount("ProgramData").path}/Nuget` + d`${Context.getMount("ProgramData").path}/Nuget`, + ...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("ProgramFilesX86").path}/Nuget`]), ], dependsOnWindowsDirectories: true, dependsOnAppDataDirectory: true, diff --git a/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc b/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc index 23fd509dc..73ddedcd0 100644 --- a/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc +++ b/Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc @@ -839,7 +839,11 @@ interface UntrackingSettings { } interface NuGetConfiguration extends ToolConfiguration { - credentialProviders?: ToolConfiguration[]; + /** + * The download timeout, in minutes, for each NuGet download pip. Defaults to 20m. + * Equivalent to configuring timeoutInMilliseconds for the corresponding Transformers.ToolDefinition + */ + downloadTimeoutMin?: number } interface ScriptResolverDefaults { diff --git a/Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc b/Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc index 3a1e53002..26048b306 100644 --- a/Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc +++ b/Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc @@ -181,9 +181,9 @@ function runTest(args : TestRunArguments) : File[] { testAssembly: args.testDeployment.primaryFile.path, qTestType: Qtest.QTestType.msTest_latest, qTestDirToDeploy: args.testDeployment.contents, - qTestAdapterPath: Transformer.sealDirectory({ + qTestAdapterPath: Transformer.sealPartialDirectory({ root: testAdapterPath, - files: globR(testAdapterPath, "*") + files: rootTestAdapterPath.contents.filter(f => f.path.isWithin(testAdapterPath)), }), qTestDotNetFramework: getQTestDotNetFramework(), qTestPlatform: Qtest.QTestPlatform.x64, diff --git a/Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs b/Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs index 6e735db49..fcc5a8c71 100644 --- a/Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs +++ b/Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs @@ -23,10 +23,24 @@ namespace BuildXL.Engine { } - /// - public void SetMountsTable(MountsTable mountsTable) + /// + /// Adds all the mounts specified in the given mounts table + /// + public void UpdateMountsTable(MountsTable mountsTable) { - m_customMountsTable = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount); + var allMounts = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount); + + if (m_customMountsTable == null) + { + m_customMountsTable = allMounts; + } + else + { + foreach (var mount in allMounts) + { + m_customMountsTable[mount.Key] = mount.Value; + } + } } /// @@ -41,7 +55,7 @@ namespace BuildXL.Engine return false; } - SetMountsTable(mountsTable); + UpdateMountsTable(mountsTable); return true; } diff --git a/Public/Src/Engine/Dll/FrontEndEngineImplementation.cs b/Public/Src/Engine/Dll/FrontEndEngineImplementation.cs index 437dcf972..274068f0c 100644 --- a/Public/Src/Engine/Dll/FrontEndEngineImplementation.cs +++ b/Public/Src/Engine/Dll/FrontEndEngineImplementation.cs @@ -93,6 +93,7 @@ namespace BuildXL.Engine private readonly MountsTable m_mountsTable; private readonly LoggingContext m_loggingContext; + private readonly DirectoryTranslator m_directoryTranslator; /// /// Creates an instance of . @@ -130,6 +131,7 @@ namespace BuildXL.Engine m_snapshotCollector = snapshotCollector; GetTimerUpdatePeriod = timerUpdatePeriod; Layout = configuration.Layout; + m_directoryTranslator = directoryTranslator; if (ShouldUseSpecCache(configuration)) { @@ -184,6 +186,12 @@ namespace BuildXL.Engine }); } + /// + public override AbsolutePath Translate(AbsolutePath path) + { + return m_directoryTranslator?.Translate(path, PathTable) ?? path; + } + private bool ShouldUseSpecCache(IConfiguration configuration) { var sourceDirectory = configuration.Layout.SourceDirectory.ToString(PathTable); diff --git a/Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs b/Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs index 0af9e12cd..2c96ffeff 100644 --- a/Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs +++ b/Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs @@ -197,7 +197,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult } - protected void AddSdk(string sdkLocation) + protected void AddSdk(string relativeSdkLocation) { Configuration.Resolvers.Add( new SourceResolverSettings @@ -206,7 +206,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult Modules = new List> { new DiscriminatingUnion( - AbsolutePath.Create(Context.PathTable, sdkLocation)), + AbsolutePath.Create(Context.PathTable, TestDeploymentDir).Combine(Context.PathTable, RelativePath.Create(Context.StringTable, relativeSdkLocation))), }, }); } diff --git a/Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs b/Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs index f240ace29..fb8ea94e9 100644 --- a/Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs +++ b/Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs @@ -35,7 +35,6 @@ namespace Test.BuildXL.Engine { private CacheInitializer m_cacheInitializer; - /* The build used for tests in this class generate the following layout * * \HelloWorldFinal.exe.config (source) diff --git a/Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc b/Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc index c303a9d93..0b000bab3 100644 --- a/Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc +++ b/Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc @@ -3,6 +3,8 @@ import * as DetoursServices from "BuildXL.Sandbox.Windows"; import * as Xml from "Sdk.Xml"; +import * as Deployment from "Sdk.Deployment"; +import {Transformer} from "Sdk.Transformers"; namespace Engine { @@ -23,8 +25,34 @@ namespace Engine { }, ]; - // Update the value of this variable if you change the version of Microsoft.Net.Compilers in config.dsc. - const microsoftNetCompilerSpec = f`${Context.getMount("FrontEnd").path}/Nuget/specs/Microsoft.Net.Compilers/4.0.1/module.config.bm`; + // We generate specs as if the compilers package was source files, but point it to the downloaded NuGet + const compilerSpecsDir = Context.getNewOutputDirectory("compilers-specs"); + const microsoftNetCompilerSpec = Transformer.writeAllText(p`${compilerSpecsDir}/module.config.bm`, + "module({name: 'Microsoft.Net.Compilers', version: '4.0.1', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences});"); + const specFile = Transformer.writeAllLines(p`${compilerSpecsDir}/package.dsc`, [ + "import {Transformer} from 'Sdk.Transformers';", + "namespace Contents {", + "export declare const qualifier: {};", + "@@public export const all: StaticDirectory = Transformer.sealPartialDirectory(d`package`, globR(d`package`, '*'));", + "}" + ]); + + // Deploy the compilers package plus the specs that refer to it + const deployable : Deployment.Definition = { + contents: [ + { + subfolder: a`compilers`, + contents: [ + { + subfolder: a`package`, + contents: [importFrom('Microsoft.Net.Compilers').Contents.all] + }, + specFile, + microsoftNetCompilerSpec + ] + } + ] + }; @@public export const categoriesToRunInParallel = [ @@ -47,12 +75,11 @@ namespace Engine { }, parallelGroups: categoriesToRunInParallel, testRunData: { - MicrosoftNetCompilersSdkLocation: microsoftNetCompilerSpec, + MicrosoftNetCompilersSdkLocation: "compilers/module.config.bm", }, tools: { exec: { dependencies: [ - microsoftNetCompilerSpec, importFrom("Microsoft.Net.Compilers").Contents.all, importFrom("Microsoft.NETCore.Compilers").Contents.all, ] @@ -87,6 +114,7 @@ namespace Engine { ], runtimeContent: [ ...libsUsedForTesting, + deployable ], }); } diff --git a/Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs b/Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs index 017e325dd..77994a616 100644 --- a/Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs +++ b/Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs @@ -882,6 +882,7 @@ namespace Test.BuildXL.Scheduler bool preservePathSetCasing = source.Vary(p => p.PreservePathSetCasing); bool writingToStandardErrorFailsExecution = source.Vary(p => p.WritingToStandardErrorFailsExecution); bool disableFullReparsePointResolving = source.Vary(p => p.DisableFullReparsePointResolving); + bool bypassFingerprintSalt = source.Vary(p => p.BypassFingerprintSalt); Process.Options options = Process.Options.None; if (hasUntrackedChildProcesses) @@ -940,6 +941,11 @@ namespace Test.BuildXL.Scheduler options |= Process.Options.DisableFullReparsePointResolving; } + if (bypassFingerprintSalt) + { + options |= Process.Options.BypassFingerprintSalt; + } + return new Process( executable: executable, workingDirectory: workingDirectory, diff --git a/Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs b/Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs index 17ccdd4b1..d5a67fb38 100644 --- a/Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs +++ b/Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs @@ -52,6 +52,7 @@ namespace BuildXL.FrontEnd.Core string weakPackageFingerprint, PackageIdentity package, AbsolutePath packageTargetFolder, + AbsolutePath pathToNuspec, Func>>> producePackage) { // Check if we can reuse a package that is layed out on disk already. @@ -97,6 +98,7 @@ namespace BuildXL.FrontEnd.Core weakPackageFingerprint, package, packageTargetFolder, + pathToNuspec, possiblePackageContents.Result), justification: "Okay to ignore putting in cache failure, will happen next time" ); @@ -208,7 +210,10 @@ namespace BuildXL.FrontEnd.Core return false; } - Logger.PackagePresumedUpToDateWithoutHashComparison(LoggingContext, package.GetFriendlyName()); + if (!FrontEndConfiguration.RespectWeakFingerprintForNugetUpToDateCheck()) + { + Logger.PackagePresumedUpToDateWithoutHashComparison(LoggingContext, package.GetFriendlyName()); + } return true; } @@ -354,7 +359,7 @@ namespace BuildXL.FrontEnd.Core // Saving package's hash file on disk. // But we can do this only when the content is not empty. - var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList(); + var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList(); if (packagesContent.Count == 0) { @@ -363,9 +368,6 @@ namespace BuildXL.FrontEnd.Core return PackageDownloadResult.RecoverableError(package); } - var newPackageHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, packagesContent); - TryUpdatePackageHashFile(loggingContext, package, packageHashFile, packageHash, newPackageHash); - m_logger.PackageRestoredFromCache(loggingContext, package.GetFriendlyName()); if (forcePopulatePackageCache) @@ -394,11 +396,20 @@ namespace BuildXL.FrontEnd.Core } } + // The hash file is stored as part of the package, so we have the package layout available there + var maybePackageHashFile = PackageHashFile.TryReadFrom(GetPackageHashFile(packageTargetFolder)); + + if (!maybePackageHashFile.Succeeded) + { + m_logger.DownloadPackageFailedDueToCacheError(loggingContext, friendlyName, maybePackageHashFile.Failure.Describe()); + return PackageDownloadResult.RecoverableError(package); + } + // Step: Return descriptor indicating it was successfully restored from the cache return PackageDownloadResult.FromCache( package, packageTargetFolder, - packageDescriptor.Contents.Select(c => RelativePath.Create(FrontEndContext.StringTable, c.Key)).ToList(), + maybePackageHashFile.Result.Content.Select(entry => RelativePath.Create(FrontEndContext.StringTable, entry)).ToList(), weakPackageFingerprintHash.Hash.ToHex()); } @@ -407,6 +418,7 @@ namespace BuildXL.FrontEnd.Core string weakPackageFingerprint, PackageIdentity package, AbsolutePath packageTargetFolder, + AbsolutePath pathToNuspec, IReadOnlyList packageContent) { var friendlyName = package.GetFriendlyName(); @@ -423,11 +435,22 @@ namespace BuildXL.FrontEnd.Core // Cache was already initialized var cache = await m_nugetCache; - // Step: Store all the files into the content cache + // Generate the hash file, since that is stored as part of the cache content + var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint); + // The content should have relative paths + var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList(); + var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content); + var packageHashFile = GetPackageHashFile(packageTargetFolder); + TryUpdatePackageHashFile(loggingContext, package, packageHashFile, oldHash: null, newHash: newHash); + + // Step: Store the nuspec and the hash file into the content cache var stringKeyedHashes = new List(); - foreach (var relativePath in packageContents) + + foreach (var absolutePath in new[] { pathToNuspec, AbsolutePath.Create(PathTable, packageHashFile)}) { - var targetFileLocation = packageTargetFolder.Combine(PathTable, relativePath).Expand(PathTable); + var targetFileLocation = absolutePath.Expand(PathTable); + var result = packageTargetFolder.TryGetRelative(PathTable, absolutePath, out var relativePath); + Contract.Assert(result); ContentHash contentHash; try @@ -453,7 +476,6 @@ namespace BuildXL.FrontEnd.Core } } - var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint); // Step: Create a descriptor and store that in the fingerprint store under the weak fingerprint. var cacheDescriptor = PackageDownloadDescriptor.Create( friendlyName, @@ -471,11 +493,6 @@ namespace BuildXL.FrontEnd.Core m_logger.PackageNotFoundInCacheAndDownloaded(loggingContext, package.Id, package.Version, weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint); - // The content should have relative paths - var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList(); - var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content); - TryUpdatePackageHashFile(loggingContext, package, GetPackageHashFile(packageTargetFolder), oldHash: null, newHash: newHash); - return Unit.Void; } diff --git a/Public/Src/FrontEnd/Core/NugetCache.cs b/Public/Src/FrontEnd/Core/NugetCache.cs index 99548e9b6..4f673203d 100644 --- a/Public/Src/FrontEnd/Core/NugetCache.cs +++ b/Public/Src/FrontEnd/Core/NugetCache.cs @@ -132,7 +132,7 @@ namespace BuildXL.FrontEnd.Core private static ContentFingerprint CreateDownloadFingerprint(string baseText) { // In case something in the cached Bond data becomes incompatible, we must not match. - const string VersionText = ", BondDataVersion=2;FingerprintVersion=1"; + const string VersionText = ", BondDataVersion=2;FingerprintVersion=5"; var fingerprint = FingerprintUtilities.Hash(baseText + VersionText); return new ContentFingerprint(fingerprint); } diff --git a/Public/Src/FrontEnd/Core/PackageHashFile.cs b/Public/Src/FrontEnd/Core/PackageHashFile.cs index b0d36d60b..47b6223af 100644 --- a/Public/Src/FrontEnd/Core/PackageHashFile.cs +++ b/Public/Src/FrontEnd/Core/PackageHashFile.cs @@ -31,7 +31,7 @@ namespace BuildXL.FrontEnd.Core // The file format change will force specs regeneration. // Change the version if the nuget spec generation has changed in a backward incompatible way. - private const string HashFileFormatVersion = "9"; + private const string HashFileFormatVersion = "10"; /// /// The minimal number of lines for the hash file. diff --git a/Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc b/Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc index 97df9362e..d6679eaf1 100644 --- a/Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc +++ b/Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc @@ -14,9 +14,11 @@ namespace Nuget { sources: globR(d`.`, "*.cs"), references: [ ...addIf(BuildXLSdk.isFullFramework, + NetFx.Netstandard.dll, + NetFx.System.IO.Compression.dll, NetFx.System.Xml.dll, NetFx.System.Xml.Linq.dll, - NetFx.Netstandard.dll + NetFx.System.Net.Http.dll ), ...addIf(BuildXLSdk.isFullFramework, importFrom("System.Memory").withQualifier({targetFramework: "netstandard2.0"}).pkg @@ -33,6 +35,7 @@ namespace Nuget { importFrom("BuildXL.Engine").Processes.dll, importFrom("BuildXL.Pips").dll, importFrom("BuildXL.Utilities").dll, + importFrom("BuildXL.Utilities").VstsAuthentication.dll, importFrom("BuildXL.Utilities").Collections.dll, importFrom("BuildXL.Utilities").Configuration.dll, importFrom("BuildXL.Utilities").Interop.dll, @@ -42,12 +45,21 @@ namespace Nuget { importFrom("BuildXL.Utilities").Script.Constants.dll, importFrom("Newtonsoft.Json").pkg, - importFrom("NuGet.Versioning").pkg, + importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg, ...BuildXLSdk.tplPackages, ], + runtimeContent: [ + // Keep in sync with path at Public\Sdk\Public\Tools\NugetDownloader\Tool.NugetDownloader.dsc + importFrom("BuildXL.Tools").NugetDownloader.dll + ], internalsVisibleTo: [ "Test.BuildXL.FrontEnd.Nuget" - ] + ], }); } diff --git a/Public/Src/FrontEnd/Nuget/Failures/NugetFailedWithNonZeroExitCodeFailure.cs b/Public/Src/FrontEnd/Nuget/Failures/NugetFailedWithNonZeroExitCodeFailure.cs deleted file mode 100644 index 4b704e3cf..000000000 --- a/Public/Src/FrontEnd/Nuget/Failures/NugetFailedWithNonZeroExitCodeFailure.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System; -using BuildXL.Utilities.Configuration; -using static BuildXL.Utilities.FormattableStringEx; -namespace BuildXL.FrontEnd.Nuget -{ - /// - /// Nuget.exe failed with non-zero exit code. - /// - public sealed class NugetFailedWithNonZeroExitCodeFailure : NugetFailure - { - private readonly INugetPackage m_package; - private readonly int m_exitCode; - private readonly string m_stdOut; - private readonly string m_stdErr; - - /// - public NugetFailedWithNonZeroExitCodeFailure(INugetPackage package, int exitCode, string stdOut, string stdErr) - : base(FailureType.PackageNotFound) - { - m_package = package; - m_exitCode = exitCode; - - m_stdOut = stdOut?.Trim(); - m_stdErr = stdErr?.Trim(); - } - - /// - public override string Describe() - { - var separator = !string.IsNullOrEmpty(m_stdOut) && !string.IsNullOrEmpty(m_stdErr) ? Environment.NewLine : string.Empty; - var output = I($"{m_stdOut}{separator}{m_stdErr}"); - return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored because nuget.exe failed with exit code '{m_exitCode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details."); - } - } -} diff --git a/Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs b/Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs index 4ec98de31..f82e117a6 100644 --- a/Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs +++ b/Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.ContractsLight; +using System.Linq; using BuildXL.Utilities; using BuildXL.Utilities.Configuration; using TypeScript.Net.Extensions; @@ -21,13 +22,21 @@ namespace BuildXL.FrontEnd.Nuget public readonly FailureType Type; /// - public readonly Exception Exception; + public readonly string Message; /// public NugetFailure(FailureType failureType, Exception e = null) { Type = failureType; - Exception = e; + + Message = GetAllMessages(e); + } + + /// + public NugetFailure(FailureType failureType, string message) + { + Type = failureType; + Message = message; } /// @@ -58,20 +67,28 @@ namespace BuildXL.FrontEnd.Nuget } /// - /// Nuget.exe failed to restore a package. + /// Nuget invocation failure /// - public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, int exitCode, string stdOut, string stdErr) + public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, Exception e) { - Contract.Requires(package != null); - Contract.Requires(exitCode != 0); + return CreateNugetInvocationFailure(package, GetAllMessages(e)); + } + + /// + /// Nuget invocation failure + /// + public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, string message) + { + Contract.RequiresNotNull(package); + Contract.RequiresNotNull(message); // If the stdOut has the following text: 'NotFound http' or 'WARNING: Unable to find version', it means that the package name or version are not found. - if (stdOut.Contains("NotFound http") || stdOut.Contains("WARNING: Unable to find version")) + if (message.Contains("NotFound http") || message.Contains("WARNING: Unable to find version")) { return new CanNotFindPackageFailure(package); } - return new NugetFailedWithNonZeroExitCodeFailure(package, exitCode, stdOut, stdErr); + return new NugetInvocationFailure(package, message); } /// @@ -89,10 +106,10 @@ namespace BuildXL.FrontEnd.Nuget { if (Package != null) { - return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Exception?.ToStringDemystified()}"); + return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Message}"); } - return I($"Failed to process nuget packages due to {Type.ToString()}. {Exception?.ToStringDemystified()}"); + return I($"Failed to process nuget packages due to {Type.ToString()}. {Message}"); } /// @@ -110,30 +127,15 @@ namespace BuildXL.FrontEnd.Nuget /// public enum FailureType { - /// - FetchNugetExe, - /// FetchCredentialProvider, - /// - WriteConfigFile, - /// WriteSpecFile, /// CleanTargetFolder, - /// - NugetFailedWithNonZeroExitCode, - - /// - NugetFailedWithIoException, - - /// - ListPackageContents, - /// ReadNuSpecFile, @@ -160,6 +162,37 @@ namespace BuildXL.FrontEnd.Nuget /// UnhandledError, + + /// + NoBaseAddressForRepository, + } + + private static string GetAllMessages(Exception e) + { + if (e is null) + { + return string.Empty; + } + + if (e is AggregateException aggregateException) + { + return string.Join(Environment.NewLine, aggregateException.Flatten().InnerExceptions.SelectMany(ie => GetInnerExceptions(ie)).Select(ex => ex.Message)); + } + else + { + return string.Join(Environment.NewLine, GetInnerExceptions(e).Select(e => e.Message)); + } + } + + private static IEnumerable GetInnerExceptions(Exception ex) + { + var innerException = ex; + do + { + yield return innerException; + innerException = innerException.InnerException; + } + while (innerException != null); } } } diff --git a/Public/Src/FrontEnd/Nuget/Failures/NugetInvocationFailure.cs b/Public/Src/FrontEnd/Nuget/Failures/NugetInvocationFailure.cs new file mode 100644 index 000000000..e559daef8 --- /dev/null +++ b/Public/Src/FrontEnd/Nuget/Failures/NugetInvocationFailure.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using BuildXL.Utilities.Configuration; +using static BuildXL.Utilities.FormattableStringEx; +namespace BuildXL.FrontEnd.Nuget +{ + /// + /// Nuget invocation failed. + /// + public sealed class NugetInvocationFailure : NugetFailure + { + private readonly INugetPackage m_package; + private readonly string m_message; + + /// + public NugetInvocationFailure(INugetPackage package, string message) + : base(FailureType.PackageNotFound) + { + m_package = package; + + m_message = message?.Trim(); + } + + /// + public override string Describe() + { + return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored. {m_message}."); + } + } +} diff --git a/Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs b/Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs index 4769091af..7e5b42a69 100644 --- a/Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs +++ b/Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs @@ -38,6 +38,14 @@ namespace BuildXL.FrontEnd.Nuget /// public MultiValueDictionary DependenciesPerFramework { get; } + + /// + /// Credential provider path that is used to retrieve the package + /// + /// + /// AbsolutePath.Invalid if none is to be used + /// + public AbsolutePath CredentialProviderPath { get; } /// public bool IsManagedPackage { get; set; } @@ -129,7 +137,8 @@ namespace BuildXL.FrontEnd.Nuget NugetFrameworkMonikers nugetFrameworkMonikers, PackageOnDisk packageOnDisk, Dictionary packagesOnConfig, - bool doNotEnforceDependencyVersions) + bool doNotEnforceDependencyVersions, + AbsolutePath credentialProviderPath) { m_context = context; PackageOnDisk = packageOnDisk; @@ -142,6 +151,7 @@ namespace BuildXL.FrontEnd.Nuget AssemblyToTargetFramework = new MultiValueDictionary(); m_dependencies = new List(); DependenciesPerFramework = new MultiValueDictionary(); + CredentialProviderPath = credentialProviderPath; } /// @@ -156,13 +166,14 @@ namespace BuildXL.FrontEnd.Nuget [CanBeNull] XDocument nuSpec, PackageOnDisk packageOnDisk, Dictionary packagesOnConfig, - bool doNotEnforceDependencyVersions) + bool doNotEnforceDependencyVersions, + AbsolutePath credentialProviderPath) { Contract.Requires(context != null); Contract.Requires(packageOnDisk != null); var analyzedPackage = new NugetAnalyzedPackage(context, nugetFrameworkMonikers, packageOnDisk, - packagesOnConfig, doNotEnforceDependencyVersions); + packagesOnConfig, doNotEnforceDependencyVersions, credentialProviderPath); analyzedPackage.ParseManagedSemantics(); if (nuSpec != null && !analyzedPackage.TryParseDependenciesFromNuSpec(nuSpec)) diff --git a/Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs b/Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs index 3070838b9..2aa9b07ea 100644 --- a/Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs +++ b/Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs @@ -41,7 +41,7 @@ namespace BuildXL.FrontEnd.Nuget public IReadOnlyCollection SupportedResolvers { get; } = new[] { WorkspaceNugetModuleResolver.NugetResolverName }; /// - public bool ShouldRestrictBuildParameters { get; } = true; + public bool ShouldRestrictBuildParameters { get; } = false; /// public void InitializeFrontEnd(FrontEndHost host, FrontEndContext context, IConfiguration configuration) diff --git a/Public/Src/FrontEnd/Nuget/NugetInspectedPackage.cs b/Public/Src/FrontEnd/Nuget/NugetInspectedPackage.cs new file mode 100644 index 000000000..64578faa7 --- /dev/null +++ b/Public/Src/FrontEnd/Nuget/NugetInspectedPackage.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.Diagnostics.ContractsLight; +using BuildXL.Utilities; + +namespace BuildXL.FrontEnd.Nuget +{ + /// + /// The result of inspecting a package with + /// + public readonly struct NugetInspectedPackage + { + /// + public readonly string Nuspec { get; } + + /// + public readonly IReadOnlyList Content { get; } + + /// + public NugetInspectedPackage(string nuspec, IReadOnlyList content) + { + Contract.RequiresNotNull(nuspec); + Contract.RequiresNotNull(content); + + Nuspec = nuspec; + Content = content; + } + } +} diff --git a/Public/Src/FrontEnd/Nuget/NugetPackageInspector.cs b/Public/Src/FrontEnd/Nuget/NugetPackageInspector.cs new file mode 100644 index 000000000..02d303881 --- /dev/null +++ b/Public/Src/FrontEnd/Nuget/NugetPackageInspector.cs @@ -0,0 +1,365 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.ContractsLight; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using BuildXL.FrontEnd.Nuget.Tracing; +using BuildXL.Utilities; +using BuildXL.Utilities.Configuration; +using BuildXL.Utilities.Instrumentation.Common; +using BuildXL.Utilities.VstsAuthentication; +using NuGet.Configuration; +using NuGet.Packaging; +using NuGet.Protocol; + +namespace BuildXL.FrontEnd.Nuget +{ + /// + /// Retrieves NuGet package layouts and specs from a set of feeds + /// + /// + /// Packages are not necessarily downloaded in full in order to inspect them, and partial downloads are always attempted first + /// + public class NugetPackageInspector + { + // A 5K chunk looks reasonable as the initial download: most package central directory fits in 5K, so no extra requests are needed + private const long MinimalChunkSizeInBytes = 5_000; + // On every partial download iteration we linearly grow using a 20K base (i.e. 20, 40, 60, 80, etc.) + private const long IncrementChunkSizeInBytes = 20_000; + + private readonly CancellationToken m_cancellationToken; + private readonly LoggingContext m_loggingContext; + private readonly IEnumerable<(string repositoryName, Uri repositoryUri)> m_repositories; + private readonly StringTable m_stringTable; + private readonly Func> m_discoverCredentialProvider; + private IReadOnlyDictionary m_packageBaseAddress; + private readonly Lazy>> m_initializationResult; + + /// + public NugetPackageInspector( + IEnumerable<(string repositoryName, Uri repositoryUri)> repositories, + StringTable stringTable, + Func> discoverCredentialProvider, + CancellationToken cancellationToken, + Utilities.Instrumentation.Common.LoggingContext loggingContext) + { + Contract.RequiresNotNull(repositories); + Contract.RequiresNotNull(discoverCredentialProvider); + + m_cancellationToken = cancellationToken; + m_loggingContext = loggingContext; + m_repositories = repositories; + m_stringTable = stringTable; + m_discoverCredentialProvider = discoverCredentialProvider; + + m_initializationResult = new Lazy>>(async () => { + + var logger = new StringBuilder(); + + try + { + return (await VSTSAuthenticationHelper.TryCreateSourceRepositories(m_repositories, m_discoverCredentialProvider, m_cancellationToken, logger)) + .Then(sourceRepositories => + { + var packageBaseAddressMutable = new Dictionary(); + foreach (var sourceRepository in sourceRepositories) + { + var serviceIndexResource = sourceRepository.GetResource(m_cancellationToken); + + if (serviceIndexResource == null) + { + return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, $"Cannot find index service for ${sourceRepository.PackageSource.SourceUri}"); + } + + foreach (Uri packageBaseAddress in serviceIndexResource.GetServiceEntryUris(ServiceTypes.PackageBaseAddress)) + { + packageBaseAddressMutable[packageBaseAddress] = sourceRepository.PackageSource.Credentials; + } + } + + m_packageBaseAddress = packageBaseAddressMutable; + + return true; + }); + + } + catch (Exception e) when (e is AggregateException || e is HttpRequestException) + { + return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, e); + } + finally + { + var log = logger.ToString(); + if (!string.IsNullOrWhiteSpace(log)) + { + Logger.Log.NuGetInspectionInitializationInfo(m_loggingContext, log); + } + } + }, + LazyThreadSafetyMode.ExecutionAndPublication); + } + + /// + /// Whether has been succesfully called + /// + /// Thread safe + public async Task IsInitializedAsync() => m_initializationResult.IsValueCreated && (await m_initializationResult.Value).Succeeded; + + /// + /// Retrieves the index sources of the specified repositories and initializes the base addresses + /// + /// Thread safe. Subsequents initializations have no effect and return the same result as the first one did. + public Task> TryInitAsync() => m_initializationResult.Value; + + /// + /// Tries to retrieve the layout and nuspec of the given package + /// + /// Thread safe + public async Task> TryInspectAsync(INugetPackage identity) + { + Contract.Assert(await IsInitializedAsync(), "TryInitAsync() must be succesfully called first"); + + if (m_packageBaseAddress.Count == 0) + { + return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository); + } + + // Build the URI for the requested package and try to inspect it + var packageIdLowerCase = identity.Id.ToLowerInvariant(); + var version = new NuGet.Versioning.NuGetVersion(identity.Version).ToNormalizedString(); + + Possible maybeInspectedPackage = default; + foreach (var baseAddress in m_packageBaseAddress) + { + // URIs for retrieving the nuspec and the nupkg + var packageUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.{version}.nupkg"; + var nuspecUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.nuspec"; + + maybeInspectedPackage = await TryInspectPackageAsync(identity, new Uri(packageUri), new Uri(nuspecUri), baseAddress.Value); + if (maybeInspectedPackage.Succeeded) + { + return maybeInspectedPackage; + } + } + + return maybeInspectedPackage.Failure; + } + + private async Task> TryInspectPackageAsync(INugetPackage identity, Uri nupkgUri, Uri nuspecUri, PackageSourceCredential packageSourceCredential) + { + AuthenticationHeaderValue authenticationHeader = null; + + // If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request + // We only want to send the token over HTTPS and to a VSTS domain to avoid security issues + if (packageSourceCredential != null) + { + authenticationHeader = VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(packageSourceCredential.PasswordText); + } + + // We want to be able to read the zip file central directory, where the layout of the package is. This is at the end of a zip file, + // so we'll start requesting partial chunks of the content starting from the end and increasingly request more until we can understand + // the zip central directory + + int retries = 3; + + while (true) + { + try + { + using (var httpClient = new HttpClient()) + { + httpClient.Timeout = TimeSpan.FromMinutes(1); + + // Use authentication if defined + if (authenticationHeader != null) + { + httpClient.DefaultRequestHeaders.Accept.Clear(); + httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + httpClient.DefaultRequestHeaders.Authorization = authenticationHeader; + } + + // We need the nuspec file for analyzing the package + var response = await httpClient.GetAsync(nuspecUri, m_cancellationToken); + + if (!response.IsSuccessStatusCode) + { + return NugetFailure.CreateNugetInvocationFailure(identity, response.ReasonPhrase); + } + + var nuspec = await response.Content.ReadAsStringAsync(); + + // Now inspect the content of the nupkg + return (await InspectContentAsync(identity, nupkgUri, authenticationHeader, httpClient)).Then(content => new NugetInspectedPackage(nuspec, content)); + } + } + catch (Exception e) when (e is HttpRequestException || e is AggregateException || e is TaskCanceledException) + { + retries--; + + if (retries == 0) + { + return NugetFailure.CreateNugetInvocationFailure(identity, e); + } + } + } + } + + private async Task>> InspectContentAsync(INugetPackage identity, Uri nupkgUri, AuthenticationHeaderValue authenticationHeader, HttpClient httpClient) + { + long? chunkStart = null; + HttpResponseMessage response; + bool forceFullDownload = false; + var partialPackage = new MemoryStream(); + + try + { + // How many chunks we downloaded so far for the given package + int chunkCount = 0; + + while (true) + { + // Only set a download range if we are not forcing a full download + if (!forceFullDownload) + { + // Set the range header to retrieve a particular range of the content + if (!chunkStart.HasValue) + { + // We don't know the total size yet, this is the first request. Start with MinimalChunkSizeInBytes. + httpClient.DefaultRequestHeaders.Add("Range", "bytes=-" + MinimalChunkSizeInBytes); + } + else + { + // This is not the first time we request a chunk, and that means the content we retrieved son far is not enough to read the zip central directory. + // So we already know where the chunk starts (and ends) + httpClient.DefaultRequestHeaders.Add("Range", $"bytes={chunkStart}-{chunkStart + (chunkCount * IncrementChunkSizeInBytes) - 1}"); + } + } + + // TODO: redirect handling may be needed here + response = await httpClient.GetAsync(nupkgUri, HttpCompletionOption.ResponseHeadersRead, m_cancellationToken); + + // In the rare case where the initial chunk is bigger than the package, we might get this. Just force full download and retry + if (response.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable) + { + httpClient.DefaultRequestHeaders.Remove("Range"); + forceFullDownload = true; + continue; + } + + if (!response.IsSuccessStatusCode) + { + break; + } + + // Check whether the service decided to ignore the partial request, and instead downloaded the whole thing + if (response.StatusCode == System.Net.HttpStatusCode.OK) + { + forceFullDownload = true; + } + + long totalLength = 0; + if (!forceFullDownload) + { + totalLength = response.Content.Headers.ContentRange.Length.Value; + + if (!chunkStart.HasValue) + { + // We just did the first request, chunk start points to MinimalChunkSizeInBytes from the end + chunkStart = totalLength - MinimalChunkSizeInBytes; + } + } +#if NET_COREAPP_60 + using (var chunk = await response.Content.ReadAsStreamAsync(m_cancellationToken)) +#else + using (var chunk = await response.Content.ReadAsStreamAsync()) +#endif + { + // Unfortunately the .net framework does not support prepending a stream, so we do it manually + // TODO: If this becomes a perf/footprint issue we could write a stream wrapper that knows how to compose streams. But + // the expectation is that we shouldn't need to iterate over the same package for very long until we can read it + partialPackage = await PrependToStreamAsync(chunk, partialPackage); + } + + // We don't want to get in the business of decoding a zip file. For that we use ZipArchive and check whether we get an exception + // when we read it. + // However, ZipArchive expects a stream with the proper length, since it does some validations about the location of the central + // directory baed on that. To avoid creating a stream of the real size (some packages are GB big), we use a ZeroPaddedStream that + // wraps the original stream but pretends to have the required size + IReadOnlyCollection entries; + try + { + var zf = new ZipArchive(forceFullDownload + ? partialPackage + : new ZeroLeftPaddedStream(partialPackage, totalLength), ZipArchiveMode.Read); + entries = zf.Entries; + } + catch (InvalidDataException ex) + { + // We downloaded the package in full but we cannot recognize it as a zip + if (forceFullDownload) + { + return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {ex.ToStringDemystified()} "); + } + + // This check is just a heuristics for the rare case when we already downloaded more than 10% of the package total + // size but we can't still read the zip central directory. At this point we can rather download the whole package + if (partialPackage.Length < totalLength * .1) + { + chunkCount++; + // We were not able to read the package central directory with what we downloaded so far. Request another chunk and + // try again + chunkStart -= chunkCount * IncrementChunkSizeInBytes; + httpClient.DefaultRequestHeaders.Remove("Range"); + + continue; + } + + httpClient.DefaultRequestHeaders.Remove("Range"); + forceFullDownload = true; + continue; + } + + return entries + .Select(entry => entry.FullName.Contains('%') ? System.Net.WebUtility.UrlDecode(entry.FullName) : entry.FullName) + .Where(entry => PackageHelper.IsPackageFile(entry, PackageSaveMode.Files | PackageSaveMode.Nuspec)) + .Select(entry => RelativePath.Create(m_stringTable, entry)).ToArray(); + } + } + finally + { + partialPackage.Dispose(); + } + + return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {response.RequestMessage} "); + } + + private async Task PrependToStreamAsync(Stream prefix, MemoryStream stream) + { + // If the destination stream is empty, just copy the prefix over + if (stream.Length == 0) + { + await prefix.CopyToAsync(stream, m_cancellationToken); + return stream; + } + + var tempPackage = new MemoryStream(); + await prefix.CopyToAsync(tempPackage, m_cancellationToken); + + stream.Position = 0; + await stream.CopyToAsync(tempPackage, m_cancellationToken); + + stream.Dispose(); + + return tempPackage; + } + } +} diff --git a/Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs b/Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs index cb9bd5bce..daa084428 100644 --- a/Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs +++ b/Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs @@ -14,6 +14,8 @@ using TypeScript.Net.Extensions; using TypeScript.Net.Types; using static TypeScript.Net.DScript.SyntaxFactory; using static BuildXL.FrontEnd.Nuget.SyntaxFactoryEx; +using BuildXL.FrontEnd.Sdk; +using BuildXL.FrontEnd.Script.Literals; namespace BuildXL.FrontEnd.Nuget { @@ -25,22 +27,31 @@ namespace BuildXL.FrontEnd.Nuget private readonly PathTable m_pathTable; private readonly PackageOnDisk m_packageOnDisk; private readonly NugetAnalyzedPackage m_analyzedPackage; - + private readonly IReadOnlyDictionary m_repositories; private readonly NugetFrameworkMonikers m_nugetFrameworkMonikers; - + private readonly AbsolutePath m_sourceDirectory; private readonly PathAtom m_xmlExtension; private readonly PathAtom m_pdbExtension; + private readonly int? m_timeoutInMinutes; /// Current spec generation format version - public const int SpecGenerationFormatVersion = 11; + public const int SpecGenerationFormatVersion = 12; /// - public NugetSpecGenerator(PathTable pathTable, NugetAnalyzedPackage analyzedPackage) + public NugetSpecGenerator( + PathTable pathTable, + NugetAnalyzedPackage analyzedPackage, + IReadOnlyDictionary repositories, + AbsolutePath sourceDirectory, + int? timeoutInMinutes = null) { m_pathTable = pathTable; m_analyzedPackage = analyzedPackage; + m_repositories = repositories; m_packageOnDisk = analyzedPackage.PackageOnDisk; m_nugetFrameworkMonikers = new NugetFrameworkMonikers(pathTable.StringTable); + m_sourceDirectory = sourceDirectory; + m_timeoutInMinutes = timeoutInMinutes; m_xmlExtension = PathAtom.Create(pathTable.StringTable, ".xml"); m_pdbExtension = PathAtom.Create(pathTable.StringTable, ".pdb"); @@ -53,13 +64,13 @@ namespace BuildXL.FrontEnd.Nuget /// The generated format is: /// [optional] import of managed sdk core /// [optional] qualifier declaration - /// const packageRoot = d`absolute path to the package roo`; /// @@public - /// export const contents: StaticDirectory = Transformer.sealDirectory( - /// packageRoot, - /// [ - /// f`${packageRoot}/file`, - /// ]); + /// export const contents: StaticDirectory = NuGetDownloader.downloadPackage( + /// { + /// id: "package ID", + /// version: "X.XX", + /// ... + /// } /// @@public /// export const pkg: NugetPackage = {contents ...}; /// @@ -68,8 +79,8 @@ namespace BuildXL.FrontEnd.Nuget { var sourceFileBuilder = new SourceFileBuilder(); - // 0. Import {Transformer} from "Sdk.Transformers" to be able to seal directories - sourceFileBuilder.Statement(ImportDeclaration(new [] { "Transformer" }, "Sdk.Transformers")); + // 0. Import * as NugetDownloader from "BuildXL.Tools.NugetDownloader" to be able to download NuGet packages + sourceFileBuilder.Statement(ImportDeclaration("NugetDownloader", "BuildXL.Tools.NugetDownloader")); // 1. Optional import of managed sdk. if (analyzedPackage.IsManagedPackage) @@ -88,12 +99,7 @@ namespace BuildXL.FrontEnd.Nuget .SemicolonAndBlankLine(); } - // 3. Declare a public directory that points to the package root, for convenience reasons - sourceFileBuilder - .Statement(new VariableDeclarationBuilder().Name("packageRoot").Initializer(PropertyAccess("Contents", "packageRoot")).Build()) - .SemicolonAndBlankLine(); - - // Create a sealed directory declaration with all the package content + // Create a seal directory declaration with all the package content sourceFileBuilder .Statement(CreatePackageContents()) .SemicolonAndBlankLine(); @@ -279,24 +285,59 @@ namespace BuildXL.FrontEnd.Nuget private IStatement CreatePackageContents() { - var relativepath = "../../../pkgs/" + m_packageOnDisk.Package.Id + "." + m_packageOnDisk.Package.Version; + // Arguments for calling the nuget downloader SDK + var downloadCallArgs = new List<(string, IExpression expression)>(4) + { + ("id", new LiteralExpression(m_analyzedPackage.Id)), + ("version", new LiteralExpression(m_analyzedPackage.Version)), + ("downloadDirectory", Identifier("outputDir")), + ("extractedFiles", new ArrayLiteralExpression(m_analyzedPackage.PackageOnDisk.Contents + .Select(relativePath => PathLikeLiteral(InterpolationKind.RelativePathInterpolation, relativePath.ToString(m_pathTable.StringTable, PathFormat.Script))))), + ("repositories", new ArrayLiteralExpression(m_repositories.Select(kvp => new ArrayLiteralExpression(new LiteralExpression(kvp.Key), new LiteralExpression(kvp.Value))))) + }; + + // If a credential provider was used to inspect the package, pass it as an argument to be able to retrieve it. + if (m_analyzedPackage.CredentialProviderPath.IsValid) + { + // If the credential provider is within the source tree, express it in terms of a mount, so the generated + // spec is more resilient to cache hits across machines + IExpression path; + if (m_sourceDirectory.TryGetRelative(m_pathTable, m_analyzedPackage.CredentialProviderPath, out var relativeCredentialProviderPath)) + { + path = PathLikeLiteral( + InterpolationKind.FileInterpolation, + new PropertyAccessExpression(new CallExpression(new PropertyAccessExpression("Context", "getMount"), new LiteralExpression("SourceRoot")), "path") , + "/" + relativeCredentialProviderPath.ToString(m_pathTable.StringTable, PathFormat.Script)); + } + else + { + path = PathLikeLiteral(InterpolationKind.FileInterpolation, m_analyzedPackage.CredentialProviderPath.ToString(m_pathTable, PathFormat.Script)); + } + + downloadCallArgs.Add(("credentialProviderPath", path)); + } + + if (m_timeoutInMinutes != null) + { + downloadCallArgs.Add(("timeoutInMinutes", new LiteralExpression(m_timeoutInMinutes.Value))); + } return new ModuleDeclaration( "Contents", Qualifier(new TypeLiteralNode()), - PathLikeConstVariableDeclaration("packageRoot", InterpolationKind.DirectoryInterpolation, relativepath, Visibility.Export), + + new VariableDeclarationBuilder().Name("outputDir").Visibility(Visibility.None).Type(new TypeReferenceNode("Directory")).Initializer( + new CallExpression(new PropertyAccessExpression("Context", "getNewOutputDirectory"), new LiteralExpression("nuget"))).Build(), + new VariableDeclarationBuilder() .Name("all") .Visibility(Visibility.Public) .Type(new TypeReferenceNode("StaticDirectory")) .Initializer( - new CallExpression( - new PropertyAccessExpression("Transformer", "sealDirectory"), - new Identifier("packageRoot"), - new ArrayLiteralExpression( - m_packageOnDisk.Contents.OrderBy(path => path.ToString(m_pathTable.StringTable)).Select(GetFileExpressionForPath) - .ToArray()))) + new CallExpression( + new PropertyAccessExpression("NugetDownloader", "downloadPackage"), + ObjectLiteral(downloadCallArgs.ToArray()))) .Build() ); } @@ -377,11 +418,10 @@ namespace BuildXL.FrontEnd.Nuget private IExpression GetFileExpressionForPath(RelativePath relativePath) { - // f`{packageRoot}/relativePath` - return PathLikeLiteral( - InterpolationKind.FileInterpolation, - Identifier("packageRoot"), - "/" + relativePath.ToString(m_pathTable.StringTable, PathFormat.Script)); + // all.assertExistence(r`relativePath`) + return new CallExpression(new PropertyAccessExpression("Contents", "all", "getFile"), PathLikeLiteral( + InterpolationKind.RelativePathInterpolation, + relativePath.ToString(m_pathTable.StringTable, PathFormat.Script))); } private IExpression CreateSimpleBinary(RelativePath binaryFile) diff --git a/Public/Src/FrontEnd/Nuget/PackageOnDisk.cs b/Public/Src/FrontEnd/Nuget/PackageOnDisk.cs index a80642e0e..6f49386b2 100644 --- a/Public/Src/FrontEnd/Nuget/PackageOnDisk.cs +++ b/Public/Src/FrontEnd/Nuget/PackageOnDisk.cs @@ -68,7 +68,9 @@ namespace BuildXL.FrontEnd.Nuget out AbsolutePath nuSpecFile, out AbsolutePath moduleConfigFile) { - nuSpecFile = AbsolutePath.Invalid; + // The nuspec file is always at the root of the package. + nuSpecFile = packageDownloadResult.TargetLocation.Combine(pathTable, nuspecFileName); + moduleConfigFile = AbsolutePath.Invalid; foreach (var relativePath in packageDownloadResult.Contents) @@ -77,11 +79,7 @@ namespace BuildXL.FrontEnd.Nuget if (!relativePath.IsEmpty) { var fileName = relativePath.GetName(); - if (fileName.CaseInsensitiveEquals(pathTable.StringTable, nuspecFileName)) - { - nuSpecFile = packageDownloadResult.TargetLocation.Combine(pathTable, relativePath); - } - else if (IsModuleConfigFileName(fileName, pathTable.StringTable)) + if (IsModuleConfigFileName(fileName, pathTable.StringTable)) { moduleConfigFile = packageDownloadResult.TargetLocation.Combine(pathTable, relativePath); } diff --git a/Public/Src/FrontEnd/Nuget/Tracing/Log.cs b/Public/Src/FrontEnd/Nuget/Tracing/Log.cs index d9bdbe9d0..3444e5028 100644 --- a/Public/Src/FrontEnd/Nuget/Tracing/Log.cs +++ b/Public/Src/FrontEnd/Nuget/Tracing/Log.cs @@ -23,33 +23,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing /// public static Logger Log { get; } = new LoggerImpl(); - [GeneratedEvent( - (ushort)LogEventId.LaunchingNugetExe, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Verbose, - Keywords = (int)Keywords.Diagnostics, - EventTask = (ushort)Tasks.Parser, - Message = "Package nuget://{id}/{version} is being restored by launching nuget.exe with commandline: {commandline}")] - public abstract void LaunchingNugetExe(LoggingContext context, string id, string version, string commandline); - - [GeneratedEvent( - (ushort)LogEventId.CredentialProviderRequiresToolUrl, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError), - EventTask = (ushort)Tasks.Parser, - Message = "CredentialProviders for the Nuget resolver are required to specify ToolUrl. '{toolName}' does not do so.")] - public abstract void CredentialProviderRequiresToolUrl(LoggingContext context, string toolName); - - [GeneratedEvent( - (ushort)LogEventId.NugetDownloadInvalidHash, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Warning, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = "Configured downloadHash '{hash}' for tool '{toolName}' is invalid. Attempt to download tool without hash guard.")] - public abstract void NugetDownloadInvalidHash(LoggingContext context, string toolName, string hash); - [GeneratedEvent( (ushort)LogEventId.NugetFailedToCleanTargetFolder, EventGenerators = EventGenerators.LocalOnly, @@ -60,74 +33,13 @@ namespace BuildXL.FrontEnd.Nuget.Tracing public abstract void NugetFailedToCleanTargetFolder(LoggingContext context, string id, string version, string targetLocation, string message); [GeneratedEvent( - (ushort)LogEventId.NugetFailedWithNonZeroExitCode, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details.")] - public abstract void NugetFailedWithNonZeroExitCode(LoggingContext context, string id, string version, int exitcode, string output); - - [GeneratedEvent( - (ushort)LogEventId.NugetFailedDueToSomeWellKnownIssue, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be restored. {message}\r\nSee the buildxl log for more details.")] - public abstract void NugetFailedDueToSomeWellKnownIssue(LoggingContext context, string id, string version, string message); - - [GeneratedEvent( - (ushort)LogEventId.NugetFailedWithNonZeroExitCodeDetailed, + (ushort)LogEventId.NugetInspectionInitialization, EventGenerators = EventGenerators.LocalOnly, EventLevel = Level.Verbose, Keywords = (ushort)Keywords.UserMessage, EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. The standard output was:\r\n{stdOut}\r\n. The standard Error was:\r\n{stdErr}")] - public abstract void NugetFailedWithNonZeroExitCodeDetailed( - LoggingContext context, - string id, - string version, - int exitcode, - string stdOut, - string stdErr); - - [GeneratedEvent( - (ushort)LogEventId.NugetFailedToListPackageContents, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be downloaded because we could not list the content of the folder '{packageFolder}': {message}")] - public abstract void NugetFailedToListPackageContents(LoggingContext context, string id, string version, string packageFolder, string message); - - [GeneratedEvent( - (ushort)LogEventId.NugetFailedToWriteConfigurationFile, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = "Could not be write configuration file to '{configFile}': {message}")] - public abstract void NugetFailedToWriteConfigurationFile(LoggingContext context, string configFile, string message); - - [GeneratedEvent( - (ushort)LogEventId.NugetFailedToWriteConfigurationFileForPackage, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be processed because we could not write configuration file to '{configFile}': {message}")] - public abstract void NugetFailedToWriteConfigurationFileForPackage( - LoggingContext context, - string id, - string version, - string configFile, - string message); + Message = "Nuget inspection info: {message}")] + public abstract void NuGetInspectionInitializationInfo(LoggingContext context, string message); [GeneratedEvent( (ushort)LogEventId.NugetFailedToWriteSpecFileForPackage, @@ -169,21 +81,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing "Package nuget://{id}/{version} could not be processed because the nuspec file cannot be found at '{expectedPath}'.")] public abstract void NugetFailedNuSpecFileNotFound(LoggingContext context, string id, string version, string expectedPath); - [GeneratedEvent( - (ushort)LogEventId.NugetFailedWithInvalidNuSpecXml, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Error, - Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError), - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be processed because the nuspec file at: '{nuspecFile}' has unexpected xml contents: {illegalXmlElement}")] - public abstract void NugetFailedWithInvalidNuSpecXml( - LoggingContext context, - string id, - string version, - string nuspecFile, - string illegalXmlElement); - [GeneratedEvent( (ushort)LogEventId.NugetPackageVersionIsInvalid, EventGenerators = EventGenerators.LocalOnly, @@ -193,16 +90,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing Message = "Invalid nuget version '{version}' found in config.dsc for package '{packageName}'. Expected version format is 'A.B.C.D'.")] public abstract void ConfigNugetPackageVersionIsInvalid(LoggingContext context, string version, string packageName); - [GeneratedEvent( - (ushort)LogEventId.NugetLaunchFailed, - EventGenerators = EventGenerators.LocalOnly, - EventLevel = Level.Verbose, - Keywords = (ushort)Keywords.UserMessage, - EventTask = (ushort)Tasks.Parser, - Message = - "Package nuget://{id}/{version} could not be downloaded because nuget.exe could not be launched: {message}")] - public abstract void NugetLaunchFailed(LoggingContext context, string id, string version, string message); - [GeneratedEvent( (ushort)LogEventId.NugetStatistics, EventGenerators = EventGenerators.LocalOnly, @@ -218,7 +105,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing EventGenerators = EventGenerators.LocalOnly, EventLevel = Level.Informational, EventTask = (ushort)Tasks.Parser, - Message = "Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).", + Message = "Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).", Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)] public abstract void NugetPackageDownloadedCount(LoggingContext context, long packagesDownloadedCount, long totalPackagesCount); @@ -227,7 +114,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing EventGenerators = EventGenerators.LocalOnly, EventLevel = Level.Informational, EventTask = (ushort)Tasks.Parser, - Message = "Restored {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.", + Message = "Inspected {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.", Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)] public abstract void NugetPackagesAreRestored(LoggingContext context, long totalPackagesCount, long totalMilliseconds); @@ -254,7 +141,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing EventGenerators = EventGenerators.LocalOnly, EventLevel = Level.Verbose, EventTask = (ushort)Tasks.Parser, - Message = "Package nuget://{id}/{version} could not be restored because of an unhandled error '{error}'.", + Message = "Package nuget://{id}/{version} could not be inspected because of an unhandled error '{error}'.", Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)] public abstract void NugetUnhandledError(LoggingContext context, string id, string version, string error); @@ -264,7 +151,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing EventLevel = Level.Informational, EventTask = (ushort)Tasks.Parser, Message = - "Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}", + "Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}", Keywords = (int)Keywords.UserMessage | (int)Keywords.OverwritableOnly)] public abstract void NugetPackageDownloadedCountWithDetails(LoggingContext context, long packagesDownloadedCount, long totalPackagesCount, string packagesToDownloadDetail); diff --git a/Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs b/Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs index 3e282e76f..84b8da6a3 100644 --- a/Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs +++ b/Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs @@ -14,40 +14,41 @@ namespace BuildXL.FrontEnd.Nuget.Tracing None = 0, // reserved 11300 .. 11400 for nuget - LaunchingNugetExe = 11300, - CredentialProviderRequiresToolUrl, - NugetDownloadInvalidHash, - NugetFailedToCleanTargetFolder, - NugetFailedWithNonZeroExitCode, - NugetFailedWithNonZeroExitCodeDetailed, - NugetFailedToListPackageContents, - NugetFailedToWriteConfigurationFile, - NugetFailedToWriteConfigurationFileForPackage, - NugetFailedToWriteSpecFileForPackage, - NugetFailedToReadNuSpecFile, - NugetFailedWithInvalidNuSpecXml, - NugetPackageVersionIsInvalid, - NugetLaunchFailed, - NugetStatistics, - NugetPackagesRestoredCount, - NugetPackagesAreRestored, - NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen, - NugetDependencyVersionWasPickedWithinRange, - NugetDependencyVersionDoesNotMatch, - NugetUnknownFramework, - NugetPackageDownloadDetails, - NugetFailedNuSpecFileNotFound, - NugetFailedDueToSomeWellKnownIssue, - NugetRegenerateNugetSpecs, - NugetRegeneratingNugetSpecs, - NugetUnhandledError, - ForcePopulateTheCacheOptionWasSpecified, - NugetConcurrencyLevel, - UsePackagesFromDisOptionWasSpecified, - NugetFailedDownloadPackagesAndGenerateSpecs, - NugetFailedDownloadPackage, - NugetFailedGenerationResultFromDownloadedPackage, - NugetFailedToWriteGeneratedSpecStateFile, - NugetCannotReuseSpecOnDisk, + // Reserved LaunchingNugetExe = 11300, + // Reserved CredentialProviderRequiresToolUrl = 11301, + // Reserved NugetDownloadInvalidHash = 11302, + NugetFailedToCleanTargetFolder = 11303, + // Reserved NugetFailedWithNonZeroExitCode = 11304, + // Reserved NugetFailedWithNonZeroExitCodeDetailed = 11305, + // Reserved NugetFailedToListPackageContents = 11306, + // Reserved NugetFailedToWriteConfigurationFile = 11307, + // Reserved NugetFailedToWriteConfigurationFileForPackage = 11308, + NugetFailedToWriteSpecFileForPackage = 11309, + NugetFailedToReadNuSpecFile = 11310, + // Reserved NugetFailedWithInvalidNuSpecXml = 11311, + NugetPackageVersionIsInvalid = 11312, + // Reserved NugetLaunchFailed = 11313, + NugetStatistics = 11314, + NugetPackagesRestoredCount = 11315, + NugetPackagesAreRestored = 11316, + NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen = 11317, + NugetDependencyVersionWasPickedWithinRange = 11318, + NugetDependencyVersionDoesNotMatch = 11319, + NugetUnknownFramework = 11320, + NugetPackageDownloadDetails = 11321, + NugetFailedNuSpecFileNotFound = 11322, + // Reserved NugetFailedDueToSomeWellKnownIssue = 11323, + NugetRegenerateNugetSpecs = 11324, + NugetRegeneratingNugetSpecs = 11325, + NugetUnhandledError = 11326, + ForcePopulateTheCacheOptionWasSpecified = 11327, + NugetConcurrencyLevel = 11328, + UsePackagesFromDisOptionWasSpecified = 11329, + NugetFailedDownloadPackagesAndGenerateSpecs = 11330, + NugetFailedDownloadPackage = 11331, + NugetFailedGenerationResultFromDownloadedPackage = 11332, + NugetFailedToWriteGeneratedSpecStateFile = 11333, + NugetCannotReuseSpecOnDisk = 11334, + NugetInspectionInitialization = 11335, } } diff --git a/Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs b/Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs index 7a325dcaf..56eeb8bcf 100644 --- a/Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs +++ b/Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs @@ -62,6 +62,8 @@ namespace BuildXL.FrontEnd.Nuget private const string SpecGenerationVersionFileSuffix = ".version"; + private const string NugetCredentialProviderEnv = "NUGET_CREDENTIALPROVIDERS_PATH"; + private NugetFrameworkMonikers m_nugetFrameworkMonikers; // These are set during Initialize @@ -82,8 +84,6 @@ namespace BuildXL.FrontEnd.Nuget private NugetResolverOutputLayout m_resolverOutputLayout; private IConfiguration m_configuration; - private readonly Lazy m_nugetToolFolder; - /// public string Kind => KnownResolverKind.NugetResolverKind; @@ -101,10 +101,6 @@ namespace BuildXL.FrontEnd.Nuget m_embeddedSpecsResolver = new WorkspaceSourceModuleResolver(stringTable, statistics, logger: null); m_useMonoBasedNuGet = OperatingSystemHelper.IsUnixOS; - - m_nugetToolFolder = new Lazy( - () => m_host.GetFolderForFrontEnd(NugetResolverName).Combine(PathTable, "nuget"), - LazyThreadSafetyMode.ExecutionAndPublication); } /// @@ -393,21 +389,58 @@ namespace BuildXL.FrontEnd.Nuget private Task> DownloadPackagesAndGenerateSpecsIfNeededInternal() { - return m_nugetGenerationResult.GetOrCreate(this, @this => @this.DownloadPackagesAndGenerateSpecsAsync()); + return m_nugetGenerationResult.GetOrCreate(this, @this => Task.FromResult(@this.DownloadPackagesAndGenerateSpecs())); } - private AbsolutePath GetNugetToolFolder() + private Possible TryResolveCredentialProvider() { - return m_nugetToolFolder.Value; + if (!m_host.Engine.TryGetBuildParameter(NugetCredentialProviderEnv, nameof(NugetFrontEnd), out string credentialProvidersPaths)) + { + return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider, $"Environment variable {NugetCredentialProviderEnv} is not set"); + } + + // Here we do something slightly simpler than what NuGet does and just look for the first credential + // provider we can find + AbsolutePath credentialProviderPath = AbsolutePath.Invalid; + foreach (string path in credentialProvidersPaths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + { + if (!AbsolutePath.TryCreate(m_context.PathTable, path, out var absolutePath)) + { + break; + } + + // Use the engine to enumerate, since the result of the enumeration should be sensitive + // to the graph building process + credentialProviderPath = m_host.Engine.EnumerateFiles(absolutePath, "CredentialProvider*.exe").FirstOrDefault(); + if (credentialProviderPath.IsValid) + { + break; + } + } + + if (!credentialProviderPath.IsValid) + { + return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider, $"Unable to authenticate using a credential provider: Credential provider was not found under '{credentialProvidersPaths}'."); + } + + // We want to rebuild the build graph if the credential provider changed. Running the whole auth process under detours sounds like too much, + // let's just read the credential provider main .exe so presence/hash gets recorded instead. + m_host.Engine.TryGetFrontEndFile(credentialProviderPath, nameof(NugetFrontEnd), out _); + + return m_host.Engine.Translate(credentialProviderPath); } - private AbsolutePath GetNugetConfigPath() + private Possible DownloadPackagesAndGenerateSpecs() { - return GetNugetToolFolder().Combine(PathTable, "NuGet.config"); - } + var maybeCredentialProviderPath = TryResolveCredentialProvider(); + + var nugetInspector = new NugetPackageInspector( + m_resolverSettings.Repositories.Select(kvp => (kvp.Key, new Uri(kvp.Value))), + PathTable.StringTable, + () => maybeCredentialProviderPath.Then(path => path.ToString(PathTable)), + m_context.CancellationToken, + m_context.LoggingContext); - private async Task> DownloadPackagesAndGenerateSpecsAsync() - { // Log if the full package restore is requested. if (m_configuration.FrontEnd.ForcePopulatePackageCache()) { @@ -429,26 +462,6 @@ namespace BuildXL.FrontEnd.Nuget using (var nugetEndToEndStopWatch = m_statistics.EndToEnd.Start()) { - var possiblePaths = await TryDownloadNugetAsync(m_resolverSettings.Configuration, GetNugetToolFolder()); - if (!possiblePaths.Succeeded) - { - Logger.Log.NugetFailedDownloadPackagesAndGenerateSpecs(m_context.LoggingContext, possiblePaths.Failure.DescribeIncludingInnerFailures()); - return possiblePaths.Failure; - } - - var possibleNugetConfig = CreateNuGetConfig(m_repositories); - - var possibleNuGetConfig = TryWriteXmlConfigFile( - package: null, - targetFile: GetNugetConfigPath(), - xmlDoc: possibleNugetConfig.Result); - - if (!possibleNuGetConfig.Succeeded) - { - Logger.Log.NugetFailedDownloadPackagesAndGenerateSpecs(m_context.LoggingContext, possibleNuGetConfig.Failure.DescribeIncludingInnerFailures()); - return possibleNuGetConfig.Failure; - } - // Will contain all packages successfully downloaded and analyzed var restoredPackagesById = new Dictionary(); @@ -468,16 +481,32 @@ namespace BuildXL.FrontEnd.Nuget { var aggregateResult = new Possible[nugetProgress.Length]; - Parallel.For(fromInclusive: 0, + if (!m_host.Engine.TryGetBuildParameter(NugetCredentialProviderEnv, nameof(NugetFrontEnd), out string allCredentialProviderPaths)) + { + allCredentialProviderPaths = string.Empty; + } + + var loopState = Parallel.For(fromInclusive: 0, toExclusive: aggregateResult.Length, new ParallelOptions() { MaxDegreeOfParallelism = concurrencyLevel, CancellationToken = m_context.CancellationToken, }, - (index) => + (index, state) => { - aggregateResult[index] = TryRestorePackageAsync(nugetProgress[index], possiblePaths.Result).GetAwaiter().GetResult(); + aggregateResult[index] = TryInspectPackageAsync( + nugetProgress[index], + maybeCredentialProviderPath.Succeeded ? maybeCredentialProviderPath.Result : AbsolutePath.Invalid, + allCredentialProviderPaths, + nugetInspector).GetAwaiter().GetResult(); + + // Let's not schedule more work in the parallel for if one of the inspections failed + if (!aggregateResult[index].Succeeded) + { + state.Break(); + } + return; }); @@ -505,6 +534,16 @@ namespace BuildXL.FrontEnd.Nuget } else { + // If the package result is null, that means there was at least one failure and the parallel for ended prematurely, but the failure + // happened after the current index + if (packageResult.Result == null) + { + // There should be at least one failure and the loop state should not be completed + Contract.Assert(!loopState.IsCompleted); + // Just skip this iteration. The failure is ahead of the current index and we'll eventually reach it + continue; + } + packageResult.Result.NugetName = nugetPackage.Package.Id; restoredPackagesById[packageResult.Result.ActualId] = packageResult.Result; } @@ -549,12 +588,12 @@ namespace BuildXL.FrontEnd.Nuget { if (m_configuration.DoesSourceDiskDriveHaveSeekPenalty(PathTable)) { - nugetConcurrency = Environment.ProcessorCount / 2; + nugetConcurrency = Environment.ProcessorCount; message = I($"Lowering restore package concurrency to {nugetConcurrency} because a source drive is on HDD."); } else { - nugetConcurrency = Math.Min(16, Environment.ProcessorCount * 2); + nugetConcurrency = Math.Min(128, Environment.ProcessorCount * 4); message = I($"Increasing restore package concurrency to {nugetConcurrency} because a source drive is on SSD."); } } @@ -564,9 +603,11 @@ namespace BuildXL.FrontEnd.Nuget return nugetConcurrency; } - private async Task> TryRestorePackageAsync( + private async Task> TryInspectPackageAsync( NugetProgress progress, - IReadOnlyList credentialProviderPaths) + AbsolutePath selectedCredentialProviderPath, + string allCredentialProviderPaths, + NugetPackageInspector nugetInspector) { progress.StartRunning(); @@ -579,11 +620,9 @@ namespace BuildXL.FrontEnd.Nuget var layout = NugetPackageOutputLayout.Create( PathTable, package, - nugetTool: credentialProviderPaths[0], - nugetConfig: GetNugetConfigPath(), resolverLayout: m_resolverOutputLayout); - var possiblePkg = await TryRestorePackageWithCache(package, progress, layout, credentialProviderPaths); + var possiblePkg = await TryInpectPackageWithCache(package, progress, layout, allCredentialProviderPaths, nugetInspector); if (!possiblePkg.Succeeded) { @@ -596,6 +635,7 @@ namespace BuildXL.FrontEnd.Nuget var analyzedPackage = AnalyzeNugetPackage( possiblePkg.Result, + selectedCredentialProviderPath, m_resolverSettings.DoNotEnforceDependencyVersions); if (!analyzedPackage.Succeeded) { @@ -780,60 +820,6 @@ namespace BuildXL.FrontEnd.Nuget return true; } - private Possible CreateNuGetConfig(IReadOnlyDictionary repositories) - { - XElement credentials = null; - if (m_useMonoBasedNuGet) - { - var localNuGetConfigPath = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), - ".config", - "NuGet", - "NuGet.Config"); - - try - { - // Sadly nuget goes all over the disk to chain configs, but when it comes to the credentials it decides not to properly merge them. - // So for now we have to hack and read the credentials from the users profile and stick them in the local config.... - if (FileUtilities.Exists(localNuGetConfigPath)) - { - ExceptionUtilities.HandleRecoverableIOException( - () => - { - var doc = XDocument.Load(localNuGetConfigPath); - credentials = doc.Element("configuration")?.Element("packageSourceCredentials"); - }, - e => throw new BuildXLException($"Failed to load nuget config {localNuGetConfigPath}", e)); - } - } - catch (BuildXLException e) - { - Logger.Log.NugetFailedToWriteConfigurationFile( - m_context.LoggingContext, - localNuGetConfigPath, - e.LogEventMessage); - return new NugetFailure(NugetFailure.FailureType.WriteConfigFile, e.InnerException); - } - } - - return new XDocument( - new XElement( - "configuration", - new XElement( - "packageRestore", - new XElement("clear"), - new XElement("add", new XAttribute("key", "enabled"), new XAttribute("value", "True"))), - new XElement( - "disabledPackageSources", - new XElement("clear")), - credentials, - new XElement( - "packageSources", - new XElement("clear"), - repositories.Select( - kv => new XElement("add", new XAttribute("key", kv.Key), new XAttribute("value", kv.Value)))))); - } - private Possible TryWriteSourceFile(INugetPackage package, AbsolutePath targetFile, ISourceFile sourceFile) { Contract.Requires(package != null); @@ -873,45 +859,6 @@ namespace BuildXL.FrontEnd.Nuget return targetFile; } - private Possible TryWriteXmlConfigFile(INugetPackage package, AbsolutePath targetFile, XDocument xmlDoc) - { - var targetFilePath = targetFile.ToString(PathTable); - - try - { - FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath)); - ExceptionUtilities.HandleRecoverableIOException( - () => - xmlDoc.Save(targetFilePath, SaveOptions.DisableFormatting), - e => - { - throw new BuildXLException("Cannot save document", e); - }); - } - catch (BuildXLException e) - { - if (package == null) - { - Logger.Log.NugetFailedToWriteConfigurationFile(m_context.LoggingContext, targetFilePath, e.LogEventMessage); - } - else - { - Logger.Log.NugetFailedToWriteConfigurationFileForPackage( - m_context.LoggingContext, - package.Id, - package.Version, - targetFilePath, - e.LogEventMessage); - } - - return new NugetFailure(package, NugetFailure.FailureType.WriteConfigFile, e.InnerException); - } - - m_host.Engine.RecordFrontEndFile(targetFile, NugetResolverName); - - return targetFile; - } - private AbsolutePath GetPackageSpecDir(NugetAnalyzedPackage analyzedPackage) { return m_resolverOutputLayout.GeneratedSpecsFolder @@ -1008,8 +955,9 @@ namespace BuildXL.FrontEnd.Nuget // No-op if the directory exists FileUtilities.CreateDirectory(packageSpecDirStr); - - var nugetSpecGenerator = new NugetSpecGenerator(PathTable, analyzedPackage); + + var nugetSpecGenerator = new NugetSpecGenerator(PathTable, analyzedPackage, m_resolverSettings.Repositories, + m_configuration.Layout.SourceDirectory, m_resolverSettings.Configuration.DownloadTimeoutMin); var possibleProjectFile = TryWriteSourceFile( analyzedPackage.PackageOnDisk.Package, @@ -1044,6 +992,7 @@ namespace BuildXL.FrontEnd.Nuget internal Possible AnalyzeNugetPackage( PackageOnDisk packageOnDisk, + AbsolutePath credentialProviderPath, bool doNotEnforceDependencyVersions) { Contract.Requires(packageOnDisk != null); @@ -1057,7 +1006,7 @@ namespace BuildXL.FrontEnd.Nuget } var result = NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_nugetFrameworkMonikers, maybeNuspecXdoc.Result, - packageOnDisk, m_packageRegistry.AllPackagesById, doNotEnforceDependencyVersions); + packageOnDisk, m_packageRegistry.AllPackagesById, doNotEnforceDependencyVersions, credentialProviderPath); if (result == null) { @@ -1119,8 +1068,7 @@ namespace BuildXL.FrontEnd.Nuget } } - - private string CreateRestoreFingerPrint(INugetPackage package, IEnumerable credentialProviderPaths) + private string CreateRestoreFingerPrint(INugetPackage package, string credentialProviderPaths) { var fingerprintParams = new List { @@ -1130,7 +1078,7 @@ namespace BuildXL.FrontEnd.Nuget }; if (credentialProviderPaths != null) { - fingerprintParams.Add("cred=" + UppercaseSortAndJoinStrings(credentialProviderPaths.Select(p => p.ToString(PathTable)))); + fingerprintParams.Add("cred=" + credentialProviderPaths); } return "nuget://" + string.Join("&", fingerprintParams); @@ -1149,11 +1097,12 @@ namespace BuildXL.FrontEnd.Nuget return restoreFingerPrint + "&" + string.Join("&", fingerprintParams); } - private async Task> TryRestorePackageWithCache( + private async Task> TryInpectPackageWithCache( INugetPackage package, NugetProgress progress, NugetPackageOutputLayout layout, - IEnumerable credentialProviderPaths) + string credentialProviderPaths, + NugetPackageInspector nugetInspector) { var packageRestoreFingerprint = CreateRestoreFingerPrint(package, credentialProviderPaths); var identity = PackageIdentity.Nuget(package.Id, package.Version, package.Alias); @@ -1165,10 +1114,23 @@ namespace BuildXL.FrontEnd.Nuget packageRestoreFingerprint, identity, layout.PackageFolder, - () => + layout.PathToNuspec, + async () => { progress.StartDownloadFromNuget(); - return TryDownloadPackage(package, layout, credentialProviderPaths); + + // We want to delay initialization until the first inspection that is actually needed + // Initializing the inspector involves resolving the index service for each specified repository + if (!await nugetInspector.IsInitializedAsync()) + { + var initResult = await nugetInspector.TryInitAsync(); + if (!initResult.Succeeded) + { + return initResult.Failure; + } + } + + return await TryInspectPackage(package, layout, nugetInspector); }); return maybePackage.Then(downloadResult => @@ -1183,34 +1145,71 @@ namespace BuildXL.FrontEnd.Nuget return string.Join(",", values.Select(s => s.ToUpperInvariant()).OrderBy(s => s)); } - private async Task>> TryDownloadPackage( - INugetPackage package, NugetPackageOutputLayout layout, IEnumerable credentialProviderPaths) + private async Task>> TryInspectPackage( + INugetPackage package, + NugetPackageOutputLayout layout, + NugetPackageInspector nugetInspector) { - var xmlConfigResult = TryWriteXmlConfigFile(package, layout.PackagesConfigFile, GetPackagesXml(package)); - if (!xmlConfigResult.Succeeded) - { - return xmlConfigResult.Failure; - } - var cleanUpResult = TryCleanupPackagesFolder(package, layout); if (!cleanUpResult.Succeeded) { return cleanUpResult.Failure; } - var nugetExeResult = await TryLaunchNugetExeAsync(package, layout, credentialProviderPaths); - if (!nugetExeResult.Succeeded) + // Inspect the package (get nuspec and layout) + var maybeInspectedPackage = await nugetInspector.TryInspectAsync(package); + if (!maybeInspectedPackage.Succeeded) { - return nugetExeResult.Failure; + return maybeInspectedPackage.Failure; } - var contentResult = TryEnumerateDirectory(package, layout.PackageDirectory); - if (!contentResult.Succeeded) - { - return contentResult.Failure; - } + var inspectedPackage = maybeInspectedPackage.Result; - return contentResult.Result; + // Serialize the nuspec to disk. In this way we can also use the bxl cache to avoid + // downloading this content again. The hash file will contain the layout, which is serialized + // later + try + { +#if NET_FRAMEWORK + return ExceptionUtilities.HandleRecoverableIOException( + () => +#else + return await ExceptionUtilities.HandleRecoverableIOException( + async () => +#endif + { + FileUtilities.CreateDirectoryWithRetry(layout.PackageFolder.ToString(PathTable)); + + // XML files need to be serialized with the right enconding, so let's use XDocument + // for that + var xdocument = XDocument.Parse(inspectedPackage.Nuspec); + + using (var nuspec = new FileStream(layout.PathToNuspec.ToString(PathTable), FileMode.Create)) + { +#if NET_FRAMEWORK + xdocument.Save(nuspec); +#else + await xdocument.SaveAsync(nuspec, SaveOptions.None, m_context.CancellationToken); +#endif + } + + return new Possible>(inspectedPackage.Content); + }, + e => + { + throw new BuildXLException("Cannot write package's nuspec file to disk", e); + }); + } + catch (BuildXLException e) + { + Logger.Log.NugetFailedToWriteSpecFileForPackage( + m_context.LoggingContext, + package.Id, + package.Version, + layout.PathToNuspec.ToString(PathTable), + e.LogEventMessage); + return new NugetFailure(package, NugetFailure.FailureType.WriteSpecFile, e.InnerException); + } } private Possible TryCleanupPackagesFolder(INugetPackage package, NugetPackageOutputLayout layout) @@ -1255,466 +1254,6 @@ namespace BuildXL.FrontEnd.Nuget } } - private async Task> TryLaunchNugetExeAsync(INugetPackage package, NugetPackageOutputLayout layout, IEnumerable credentialProviderPaths) - { - var fileAccessManifest = GenerateFileAccessManifest(layout, credentialProviderPaths); - - var buildParameters = BuildParameters - .GetFactory() - .PopulateFromEnvironment(); - - var tool = layout.NugetTool.ToString(PathTable); - - var argumentsBuilder = new StringBuilder(); - if (m_useMonoBasedNuGet) - { - argumentsBuilder.AppendFormat("\"{0}\"", tool); - argumentsBuilder.Append(" "); - - if (!buildParameters.ToDictionary().TryGetValue("MONO_HOME", out var monoHome)) - { - return new NugetFailure(package, NugetFailure.FailureType.MissingMonoHome); - } - - tool = Path.Combine(monoHome, "mono"); - } - - // TODO:escape quotes properly - argumentsBuilder - .AppendFormat("restore \"{0}\"", layout.PackagesConfigFile.ToString(PathTable)) - .AppendFormat(" -OutputDirectory \"{0}\"", layout.PackageRootFolder.ToString(PathTable)) - .Append(" -Verbosity detailed") - .AppendFormat(" -ConfigFile \"{0}\"", layout.NugetConfig.ToString(PathTable)) - .Append(" -PackageSaveMode nuspec") - .Append(" -NoCache") - // Currently we have to hack nuget to MsBuild version 4 which should come form the current CLR. - .Append(" -MsBuildVersion 4"); - - if (!m_host.Configuration.Interactive) - { - // Prevent Nuget from showing any UI when not in interactive mode - argumentsBuilder.Append(" -NonInteractive"); - } - - var arguments = argumentsBuilder.ToString(); - - Logger.Log.LaunchingNugetExe(m_context.LoggingContext, package.Id, package.Version, tool + " " + arguments); - try - { - // For NugetFrontEnd always create a new ConHost process. - // The NugetFrontEnd is normally executed only once, so the overhead is low. - // Also the NugetFrontEnd is a really long running process, so creating the ConHost is relatively - // very cheap. It provides guarantee if the process pollutes the ConHost env, - // it will not affect the server ConHost. - var info = - new SandboxedProcessInfo( - m_context.PathTable, - new NugetFileStorage(layout.PackageTmpDirectory), - tool, - fileAccessManifest, - disableConHostSharing: true, - ContainerConfiguration.DisabledIsolation, - loggingContext: m_context.LoggingContext, - sandboxConnection: m_useMonoBasedNuGet ? new SandboxConnectionFake() : null) - { - Arguments = arguments, - WorkingDirectory = layout.TempDirectory.ToString(PathTable), - PipSemiStableHash = 0, - PipDescription = "NuGet FrontEnd", - EnvironmentVariables = GetNugetEnvironmentVariables(), - Timeout = TimeSpan.FromMinutes(20), // Limit the time nuget has to download each nuget package - }; - - return await RetryOnFailure( - runNuget: async () => - { - var process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing: !m_useMonoBasedNuGet); - var result = await process.GetResultAsync(); - return (result, result.ExitCode == 0); - }, - onError: async result => - { - // Log the result before trying again - var (stdOut, stdErr) = await GetStandardOutAndError(result); - - Logger.Log.NugetFailedWithNonZeroExitCodeDetailed( - m_context.LoggingContext, - package.Id, - package.Version, - result.ExitCode, - stdOut, - stdErr); - - return (stdOut, stdErr); - }, - onFinalFailure: (exitCode, stdOut, stdErr) => - { - // Give up and fail - return NugetFailure.CreateNugetInvocationFailure(package, exitCode, stdOut, stdErr); - }); - } - catch (BuildXLException e) - { - Logger.Log.NugetLaunchFailed(m_context.LoggingContext, package.Id, package.Version, e.LogEventMessage); - return new NugetFailure(package, NugetFailure.FailureType.NugetFailedWithIoException); - } - catch (Exception e) - { - return new NugetFailure(package, NugetFailure.FailureType.NugetFailedWithIoException, e); - } - - async Task<(string stdOut, string stdErr)> GetStandardOutAndError(SandboxedProcessResult result) - { - try - { - await result.StandardOutput.SaveAsync(); - var stdOut = await result.StandardOutput.ReadValueAsync(); - - await result.StandardError.SaveAsync(); - var stdErr = await result.StandardError.ReadValueAsync(); - - return (stdOut, stdErr); - } - catch (BuildXLException e) - { - return (e.LogEventMessage, string.Empty); - } - } - - BuildParameters.IBuildParameters GetNugetEnvironmentVariables() - { - // the environment variable names below should use the casing appropriate for the target OS - // (on Windows it won't matter, but on Unix-like systems, including Cygwin environment on Windows, - // it matters, and has to be all upper-cased). See also doc comment for IBuildParameters.Select - return buildParameters - .Select( - new[] - { - "ComSpec", - "PATH", - "PATHEXT", - "NUMBER_OF_PROCESSORS", - "OS", - "PROCESSOR_ARCHITECTURE", - "PROCESSOR_IDENTIFIER", - "PROCESSOR_LEVEL", - "PROCESSOR_REVISION", - "SystemDrive", - "SystemRoot", - "SYSTEMTYPE", - "NUGET_CREDENTIALPROVIDERS_PATH", - "__CLOUDBUILD_AUTH_HELPER_CONFIG__", - "__Q_DPAPI_Secrets_Dir", - - // Nuget Credential Provider env variables - "1ESSHAREDASSETS_BUILDXL_FEED_PAT", - "CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT", - - // Auth material needed for low-privilege build. - "QAUTHMATERIALROOT", - - // Used by the artifacts credential provider. See here for more information on how this variable is configured - https://github.com/microsoft/artifacts-credprovider#azure-devops-server - "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS" - }) - .Override( - new Dictionary() - { - {"TMP", layout.TempDirectoryAsString}, - {"TEMP", layout.TempDirectoryAsString}, - {"NUGET_PACKAGES", layout.TempDirectoryAsString}, - {"NUGET_ROOT", layout.TempDirectoryAsString}, - }); - } - } - - private class SandboxConnectionFake : ISandboxConnection - { - public SandboxKind Kind => SandboxKind.MacOsKext; - - public int NumberOfKextConnections => 1; - - public ulong MinReportQueueEnqueueTime { get; set; } - - public TimeSpan CurrentDrought - { - get - { - var nowNs = Sandbox.GetMachAbsoluteTime(); - var minReportTimeNs = MinReportQueueEnqueueTime; - return TimeSpan.FromTicks(nowNs > minReportTimeNs ? (long)((nowNs - minReportTimeNs) / 100) : 0); - } - } - - public void Dispose() { } - - public bool IsInTestMode => true; - - public bool NotifyUsage(uint cpuUsage, uint availableRamMB) { return true; } - - public bool NotifyPipStarted(LoggingContext loggingContext, FileAccessManifest fam, SandboxedProcessUnix process) { return true; } - - public IEnumerable<(string, string)> AdditionalEnvVarsToSet(long pipId) - { - return Enumerable.Empty<(string, string)>(); - } - - public void NotifyPipProcessTerminated(long pipId, int processId) { } - - public void NotifyRootProcessExited(long pipId, SandboxedProcessUnix process) { } - - public bool NotifyPipFinished(long pipId, SandboxedProcessUnix process) { return true; } - - public void ReleaseResources() { } - } - - private static async Task> RetryOnFailure( - Func> runNuget, - Func> onError, - Func> onFinalFailure, - int retryCount = MaxRetryCount, - int retryDelayMs = RetryDelayMs) - { - Contract.Assert(retryCount >= 1, "Maximum retry count must be greater than or equal to one. Found " + retryCount); - - var pass = false; - var iteration = 1; - - while (!pass) - { - var tuple = await runNuget(); - var result = tuple.result; - pass = tuple.isPassed; - - if (!pass) - { - var (stdOut, stdErr) = await onError(result); - - if (iteration >= retryCount) - { - return onFinalFailure(result.ExitCode, stdOut, stdErr); - } - - // Try again! - iteration++; - - await Task.Delay(retryDelayMs); - } - } - - return Unit.Void; - } - - private FileAccessManifest GenerateFileAccessManifest(NugetPackageOutputLayout layout, IEnumerable credentialProviderPaths) - { - var fileAccessManifest = new FileAccessManifest(PathTable) - { - // TODO: If this is set to true, then NuGet will fail if TMG Forefront client is running. - // Filtering out in SandboxedProcessReport won't work because Detours already blocks the access to FwcWsp.dll. - // Almost all machines in Office run TMG Forefront client. - // So far for WDG, FailUnexpectedFileAccesses is false due to allowlists. - // As a consequence, the file access manifest below gets nullified. - FailUnexpectedFileAccesses = false, - ReportFileAccesses = true, - MonitorNtCreateFile = true, - MonitorZwCreateOpenQueryFile = true, - }; - - fileAccessManifest.AddScope(layout.TempDirectory, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - fileAccessManifest.AddScope(layout.PackageRootFolder, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - if (!OperatingSystemHelper.IsUnixOS) - { - fileAccessManifest.AddScope( - AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows)), - FileAccessPolicy.MaskAll, - FileAccessPolicy.AllowAllButSymlinkCreation); - } - - fileAccessManifest.AddPath(layout.NugetTool, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing); - fileAccessManifest.AddPath(layout.NugetToolExeConfig, values: FileAccessPolicy.AllowReadIfNonexistent, mask: FileAccessPolicy.MaskNothing); - fileAccessManifest.AddPath(layout.NugetConfig, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing); - fileAccessManifest.AddPath(layout.PackagesConfigFile, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing); - - // Nuget is picky - fileAccessManifest.AddScope(layout.ResolverFolder, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - - // Nuget fails if it can't access files in the users (https://github.com/NuGet/Home/issues/2676) profile. - // We'll have to explicitly set all config in our nuget.config file and override anything the user can set and allow the read here. - var roamingAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.ApplicationData)).Combine(PathTable, "NuGet"); - fileAccessManifest.AddScope(roamingAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - - var localAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)).Combine(PathTable, "NuGet"); - fileAccessManifest.AddScope(localAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - - // Nuget also probes in ProgramData on the machine. - var commonAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.CommonApplicationData)).Combine(PathTable, "NuGet"); - fileAccessManifest.AddScope(commonAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation); - - foreach (var providerPath in credentialProviderPaths) - { - fileAccessManifest.AddPath(providerPath, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing); - fileAccessManifest.AddPath( - providerPath.ChangeExtension(PathTable, PathAtom.Create(PathTable.StringTable, ".exe.config")), - values: FileAccessPolicy.AllowRead, - mask: FileAccessPolicy.MaskNothing); - } - - return fileAccessManifest; - } - - private static XDocument GetPackagesXml(INugetPackage package) - { - - var version = package.Version; - var plusIndex = version.IndexOf('+'); - if (plusIndex > 0) - { - version = version.Substring(0, plusIndex); - } - return new XDocument( - new XElement( - "packages", - new XElement( - "package", - new XAttribute("id", package.Id), - new XAttribute("version", version)))); - } - - private Possible, NugetFailure> TryEnumerateDirectory(INugetPackage package, string packagePath) - { - var enumerateDirectoryResult = EnumerateDirectoryRecursively(packagePath, out var contents); - - if (!enumerateDirectoryResult.Succeeded) - { - var message = enumerateDirectoryResult.GetNativeErrorMessage(); - Logger.Log.NugetFailedToListPackageContents(m_context.LoggingContext, package.Id, package.Version, packagePath, message); - return new NugetFailure(package, NugetFailure.FailureType.ListPackageContents, enumerateDirectoryResult.CreateExceptionForError()); - } - - return contents; - } - - private EnumerateDirectoryResult EnumerateDirectoryRecursively(string packagePath, out List resultingContent) - { - resultingContent = new List(); - return EnumerateDirectoryRecursively(RelativePath.Empty, resultingContent); - - EnumerateDirectoryResult EnumerateDirectoryRecursively(RelativePath relativePath, List contents) - { - var result = FileUtilities.EnumerateDirectoryEntries( - Path.Combine(packagePath, relativePath.ToString(m_context.StringTable)), - (name, attr) => - { - var nestedRelativePath = relativePath.Combine(PathAtom.Create(m_context.StringTable, name)); - if ((attr & FileAttributes.Directory) != 0) - { - EnumerateDirectoryRecursively(nestedRelativePath, contents); - } - else - { - contents.Add(nestedRelativePath); - } - }); - - return result; - } - } - - private async Task> TryDownloadNugetAsync(INugetConfiguration configuration, AbsolutePath targetFolder) - { - configuration = configuration ?? new NugetConfiguration(); - - var downloads = new Task>[1 + configuration.CredentialProviders.Count]; - var paths = new AbsolutePath[downloads.Length]; - - var nugetTargetLocation = targetFolder.Combine(m_context.PathTable, "nuget.exe"); - - var nugetLocation = configuration.ToolUrl; - if (string.IsNullOrEmpty(nugetLocation)) - { - var version = configuration.Version ?? "latest"; - nugetLocation = string.Format(CultureInfo.InvariantCulture, "https://dist.nuget.org/win-x86-commandline/{0}/nuget.exe", version); - } - - TryGetExpectedContentHash(configuration, out var expectedHash); - - downloads[0] = m_host.DownloadFile(nugetLocation, nugetTargetLocation, expectedHash, NugetResolverName); - paths[0] = nugetTargetLocation; - - for (var i = 0; i < configuration.CredentialProviders.Count; i++) - { - var credentialProvider = configuration.CredentialProviders[i]; - var credentialProviderName = NugetResolverName + ".credentialProvider." + i.ToString(CultureInfo.InvariantCulture); - - TryGetExpectedContentHash(credentialProvider, out var expectedProviderHash); - - var toolUrl = credentialProvider.ToolUrl; - if (string.IsNullOrEmpty(toolUrl)) - { - // TODO: Have better provenance for configuration values. - Logger.Log.CredentialProviderRequiresToolUrl(m_context.LoggingContext, credentialProviderName); - return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider); - } - - var fileNameStart = toolUrl.LastIndexOfAny(new[] { '/', '\\' }); - var fileName = fileNameStart >= 0 ? toolUrl.Substring(fileNameStart + 1) : toolUrl; - var targetLocation = targetFolder.Combine(m_context.PathTable, fileName); - downloads[i + 1] = m_host.DownloadFile(toolUrl, targetLocation, expectedProviderHash, credentialProviderName); - paths[i + 1] = targetLocation; - } - - var results = await Task.WhenAll(downloads); - - foreach (var result in results) - { - if (!result.Succeeded) - { - return result.Failure; - } - } - - return paths; - } - - /// - private bool TryGetExpectedContentHash(IArtifactLocation artifactLocation, out ContentHash? expectedHash) - { - expectedHash = null; - if (!string.IsNullOrEmpty(artifactLocation.Hash)) - { - if (!ContentHash.TryParse(artifactLocation.Hash, out var contentHash)) - { - // TODO: better provenance for configuration settings. - Logger.Log.NugetDownloadInvalidHash( - m_context.LoggingContext, - "nuget.exe", - artifactLocation.Hash); - return false; - } - - expectedHash = contentHash; - } - - return true; - } - - /// - private sealed class NugetFileStorage : ISandboxedProcessFileStorage - { - private readonly string m_directory; - - /// - public NugetFileStorage(string directory) - { - m_directory = directory; - } - - /// - public string GetFileName(SandboxedProcessFile file) - { - return Path.Combine(m_directory, file.DefaultFileName()); - } - } - /// /// Helper class that holds all folders required for a nuget resolver /// @@ -1753,50 +1292,30 @@ namespace BuildXL.FrontEnd.Nuget { private readonly NugetResolverOutputLayout m_resolverLayout; - public NugetPackageOutputLayout(PathTable pathTable, INugetPackage package, AbsolutePath nugetTool, AbsolutePath nugetConfig, NugetResolverOutputLayout resolverLayout) + public NugetPackageOutputLayout(PathTable pathTable, INugetPackage package, NugetResolverOutputLayout resolverLayout) { m_resolverLayout = resolverLayout; - Package = package; - NugetTool = nugetTool; - NugetConfig = nugetConfig; var idAndVersion = package.Id + "." + package.Version; - NugetToolExeConfig = nugetTool.ChangeExtension(pathTable, PathAtom.Create(pathTable.StringTable, ".exe.config")); // All the folders should include version to avoid potential race conditions during nuget execution. PackageFolder = PackageRootFolder.Combine(pathTable, idAndVersion); PackageDirectory = PackageFolder.ToString(pathTable); PackageTmpDirectory = TempDirectory.Combine(pathTable, idAndVersion).ToString(pathTable); - PackagesConfigFile = ConfigRootFolder.Combine(pathTable, idAndVersion).Combine(pathTable, "packages.config"); + PathToNuspec = PackageFolder.Combine(pathTable, $"{package.Id}.nuspec"); } public static NugetPackageOutputLayout Create(PathTable pathTable, INugetPackage package, - AbsolutePath nugetTool, AbsolutePath nugetConfig, NugetResolverOutputLayout resolverLayout) + NugetResolverOutputLayout resolverLayout) { - return new NugetPackageOutputLayout(pathTable, package, nugetTool, nugetConfig, resolverLayout); + return new NugetPackageOutputLayout(pathTable, package, resolverLayout); } - public INugetPackage Package { get; } - - public AbsolutePath ResolverFolder => m_resolverLayout.ResolverFolder; - - public string ResolverDirectory => m_resolverLayout.ResolverDirectory; - - public AbsolutePath NugetTool { get; } - - public AbsolutePath NugetToolExeConfig { get; } - - public AbsolutePath NugetConfig { get; } - public AbsolutePath TempDirectory => m_resolverLayout.TempDirectory; - public string TempDirectoryAsString => m_resolverLayout.TempDirectoryAsString; - public AbsolutePath PackageRootFolder => m_resolverLayout.PackageRootFolder; - public AbsolutePath ConfigRootFolder => m_resolverLayout.ConfigRootFolder; - - public AbsolutePath PackagesConfigFile { get; } + public AbsolutePath PathToNuspec { get; } public AbsolutePath PackageFolder { get; } diff --git a/Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs b/Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs index ddb585f96..ed822ace9 100644 --- a/Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs +++ b/Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs @@ -181,6 +181,8 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers private SymbolAtom m_semaphoreInfoLimit; private SymbolAtom m_semaphoreInfoName; private SymbolAtom m_semaphoreInfoIncrementBy; + // Explicitly not exposed in DScript since bypassing the salts is not officially supported, and only used for internally scheduled pips + private SymbolAtom m_unsafeBypassFingerprintSalt; private CallSignature ExecuteSignature => CreateSignature( required: RequiredParameters(AmbientTypes.ExecuteArgumentsType), @@ -332,6 +334,7 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers m_unsafeTrustStaticallyDeclaredAccesses = Symbol("trustStaticallyDeclaredAccesses"); m_unsafeDisableFullReparsePointResolving = Symbol("disableFullReparsePointResolving"); m_unsafeDisableSandboxing = Symbol("disableSandboxing"); + m_unsafeBypassFingerprintSalt = Symbol("bypassFingerprintSalt"); // Semaphore info. m_semaphoreInfoLimit = Symbol("limit"); @@ -1416,6 +1419,12 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers { processBuilder.Options |= Process.Options.DisableSandboxing; } + + // UnsafeExecuteArguments.bypassFingerprintSalt + if (Converter.ExtractOptionalBoolean(unsafeOptionsObjLit, m_unsafeBypassFingerprintSalt) == true) + { + processBuilder.Options |= Process.Options.BypassFingerprintSalt; + } } private PipId InterpretFinalizationPipArguments(Context context, ObjectLiteral obj) diff --git a/Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs b/Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs index 27a62361c..b8168762f 100644 --- a/Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs +++ b/Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs @@ -222,6 +222,11 @@ namespace BuildXL.FrontEnd.Sdk return EnumerateEntries(path, pattern, recursive, directories: true); } + /// + /// Translates an absolute path according to the defined translations + /// + public abstract AbsolutePath Translate(AbsolutePath path); + /// /// A simple IO based file system entry enumeration helper /// diff --git a/Public/Src/FrontEnd/Sdk/FrontEndHost.cs b/Public/Src/FrontEnd/Sdk/FrontEndHost.cs index 5e696d3cf..9aee2fae5 100644 --- a/Public/Src/FrontEnd/Sdk/FrontEndHost.cs +++ b/Public/Src/FrontEnd/Sdk/FrontEndHost.cs @@ -121,6 +121,7 @@ namespace BuildXL.FrontEnd.Sdk string weakPackageFingerprint, PackageIdentity package, AbsolutePath packageTargetFolder, + AbsolutePath pathToNuspec, Func>>> producePackage); /// diff --git a/Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs b/Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs index 5aaa1987e..cd8d847e4 100644 --- a/Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs +++ b/Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs @@ -65,6 +65,12 @@ namespace BuildXL.FrontEnd.Sdk }; } + /// + public override AbsolutePath Translate(AbsolutePath path) + { + return path; + } + /// /// This method is not implemented. /// @@ -86,7 +92,7 @@ namespace BuildXL.FrontEnd.Sdk return false; } - stream = m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream; + stream = m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream; return true; } diff --git a/Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs b/Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs index 017a9e88c..15cfd3d5e 100644 --- a/Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs +++ b/Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs @@ -42,6 +42,12 @@ namespace BuildXL.FrontEnd.Script.Testing.Helper }; } + /// + public override AbsolutePath Translate(AbsolutePath path) + { + return path; + } + /// public override bool TryGetFrontEndFile(AbsolutePath path, string frontEnd, out Stream stream) { diff --git a/Public/Src/FrontEnd/UnitTests/Core/DsTest.cs b/Public/Src/FrontEnd/UnitTests/Core/DsTest.cs index 0efb4a63f..902a87937 100644 --- a/Public/Src/FrontEnd/UnitTests/Core/DsTest.cs +++ b/Public/Src/FrontEnd/UnitTests/Core/DsTest.cs @@ -30,6 +30,7 @@ using BuildXL.FrontEnd.Workspaces.Core; using BuildXL.Native.IO; using BuildXL.Pips.Filter; using BuildXL.Utilities; +using BuildXL.Utilities.Collections; using BuildXL.Utilities.Configuration; using BuildXL.Utilities.Configuration.Mutable; using BuildXL.Utilities.Instrumentation.Common; @@ -1074,6 +1075,8 @@ namespace Test.BuildXL.FrontEnd.Core frontEndFactory.InitializeFrontEnds(controller, FrontEndContext, engine.Configuration); var frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(FrontEndContext.PathTable, engineContext.FileSystem, engine.Configuration); + frontEndEngineAbstraction.TryPopulateWithDefaultMountsTable(LoggingContext, engineContext, engine.Configuration, CollectionUtilities.EmptyDictionary()); + controller.SetState(frontEndEngineAbstraction, GetPipGraph(), config); var evaluationFilter = fileToProcess.IsValid ? EvaluationFilter.FromSingleSpecPath(FrontEndContext.SymbolTable, FrontEndContext.PathTable, fileToProcess) : EvaluationFilter.Empty; diff --git a/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Context.ts b/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Context.ts index 86a39be29..86f6cefe5 100644 --- a/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Context.ts +++ b/Public/Src/FrontEnd/UnitTests/Libs/Prelude.Context.ts @@ -98,7 +98,7 @@ namespace Context { */ interface CurrentHostInformation { /** The current Os type */ - os: "win" | "mac" | "unix"; + os: "win" | "macOS" | "unix"; /** The current cpu architecture */ cpuArchitecture: "x64" | "x86"; /** Wheter the current build is run with elevated permissions (admin/sudo) */ diff --git a/Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs b/Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs index 946c60db1..0687193a1 100644 --- a/Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs +++ b/Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs @@ -11,17 +11,18 @@ using Xunit; using Xunit.Abstractions; using Test.BuildXL.TestUtilities.Xunit; using System; +using BuildXL.Utilities; namespace Test.BuildXL.FrontEnd.Nuget { public class NuSpecGeneratorTests { - private const int CurrentSpecGenVersion = 11; + private const int CurrentSpecGenVersion = 12; private readonly ITestOutputHelper m_output; private readonly FrontEndContext m_context; private readonly PackageGenerator m_packageGenerator; - + private readonly Dictionary m_repositories; private static readonly INugetPackage s_myPackage = new NugetPackage() { Id = "MyPkg", Version = "1.99" }; private static readonly INugetPackage s_systemCollections = new NugetPackage() { Id = "System.Collections", Version = "4.0.11" }; private static readonly INugetPackage s_systemCollectionsConcurrent = new NugetPackage() { Id = "System.Collections.Concurrent", Version = "4.0.12" }; @@ -42,6 +43,8 @@ namespace Test.BuildXL.FrontEnd.Nuget var monikers = new NugetFrameworkMonikers(m_context.StringTable); m_packageGenerator = new PackageGenerator(m_context, monikers); + + m_repositories = new Dictionary() { ["BuildXL"] = "https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json" }; } [Fact] @@ -70,11 +73,11 @@ namespace Test.BuildXL.FrontEnd.Nuget ", s_packagesOnConfig, new string[] { "lib/net45/my.dll", "lib/net451/my.dll", "lib/netstandard2.0/my.dll"}); - var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg); + var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg); var text = spec.ToDisplayStringV2(); m_output.WriteLine(text); - string expectedSpec = $@"import {{Transformer}} from ""Sdk.Transformers""; + string expectedSpec = $@"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader""; import * as Managed from ""Sdk.Managed""; export declare const qualifier: {{ @@ -82,22 +85,23 @@ export declare const qualifier: {{ targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"", }}; -const packageRoot = Contents.packageRoot; - namespace Contents {{ export declare const qualifier: {{ }}; - export const packageRoot = d`../../../pkgs/TestPkg.1.999`; + const outputDir: Directory = Context.getNewOutputDirectory(""nuget""); @@public - export const all: StaticDirectory = Transformer.sealDirectory( - packageRoot, - [ - f`${{packageRoot}}/lib/net45/my.dll`, - f`${{packageRoot}}/lib/net451/my.dll`, - f`${{packageRoot}}/lib/netstandard2.0/my.dll`, - f`${{packageRoot}}/TestPkg.nuspec`, - ] - ); + export const all: StaticDirectory = NugetDownloader.downloadPackage({{ + id: ""TestPkg"", + version: ""1.999"", + downloadDirectory: outputDir, + extractedFiles: [ + r`TestPkg.nuspec`, + r`lib/net45/my.dll`, + r`lib/net451/my.dll`, + r`lib/netstandard2.0/my.dll`, + ], + repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]], + }}); }} @@public @@ -108,9 +112,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{ ""TestPkg"", ""1.999"", Contents.all, - [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)], - [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)], - [importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg] + [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))], + [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))], + [ + importFrom(""System.Collections"").pkg, + importFrom(""System.Collections.Concurrent"").pkg, + ] ); case ""net451"": case ""net452"": @@ -122,9 +129,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{ ""TestPkg"", ""1.999"", Contents.all, - [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)], - [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)], - [importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg] + [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))], + [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))], + [ + importFrom(""System.Collections"").pkg, + importFrom(""System.Collections.Concurrent"").pkg, + ] ); case ""netstandard2.0"": case ""netcoreapp2.0"": @@ -139,9 +149,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{ ""TestPkg"", ""1.999"", Contents.all, - [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`)], + [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`))], [ - Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`), + Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`)), ], [...addIfLazy(qualifier.targetFramework === ""netstandard2.0"", () => [importFrom(""Newtonsoft.Json"").pkg])] ); @@ -150,9 +160,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{ }}; }} )();"; - XAssert.AreEqual(expectedSpec, text); + XAssert.AreEqual(expectedSpec.Trim(), text.Trim()); - const string CurrentSpecHash = "FDBA16F722AB64B5C61F4CBAD40500C9B1944E39"; + const string CurrentSpecHash = "1291F51E8E59ED9C2F967C01D49CC39FE6DEB9D8"; ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash); } @@ -160,25 +170,29 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{ public void GenerateNuSpecForStub() { var pkg = m_packageGenerator.AnalyzePackageStub(s_packagesOnConfig); - var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg); + var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg); var text = spec.ToDisplayStringV2(); m_output.WriteLine(text); - string expectedSpec = @"import {Transformer} from ""Sdk.Transformers""; + string expectedSpec = @"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader""; export declare const qualifier: { targetFramework: ""net10"" | ""net11"" | ""net20"" | ""net35"" | ""net40"" | ""net45"" | ""net451"" | ""net452"" | ""net46"" | ""net461"" | ""net462"" | ""net472"" | ""netstandard1.0"" | ""netstandard1.1"" | ""netstandard1.2"" | ""netstandard1.3"" | ""netstandard1.4"" | ""netstandard1.5"" | ""netstandard1.6"" | ""netstandard2.0"" | ""netcoreapp2.0"" | ""netcoreapp2.1"" | ""netcoreapp2.2"" | ""netcoreapp3.0"" | ""netcoreapp3.1"" | ""net5.0"" | ""net6.0"" | ""netstandard2.1"", targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"", }; -const packageRoot = Contents.packageRoot; - namespace Contents { export declare const qualifier: { }; - export const packageRoot = d`../../../pkgs/TestPkgStub.1.999`; + const outputDir: Directory = Context.getNewOutputDirectory(""nuget""); @@public - export const all: StaticDirectory = Transformer.sealDirectory(packageRoot, []); + export const all: StaticDirectory = NugetDownloader.downloadPackage({ + id: ""TestPkgStub"", + version: ""1.999"", + downloadDirectory: outputDir, + extractedFiles: [], + repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]], + }); } @@public @@ -189,7 +203,7 @@ export const pkg: NugetPackage = { };"; XAssert.ArrayEqual(SplitToLines(expectedSpec), SplitToLines(text)); - const string CurrentSpecHash = "8550BF2B35888E3C0095095F44E2EB06FC2F0576"; + const string CurrentSpecHash = "389BC336BD4B206394E01B1E76A859E6561CE30D"; ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash); } diff --git a/Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs b/Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs index 22da60f0b..7b671cada 100644 --- a/Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs +++ b/Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs @@ -64,7 +64,7 @@ namespace Test.BuildXL.FrontEnd.Nuget var nugetResolver = CreateWorkspaceNugetResolverForTesting(); var allPackages = new Dictionary { [packageOnDisk.Package.Id] = packageOnDisk.Package }; - var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false); + var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false); XAssert.IsTrue(analyzedPackage.Succeeded); @@ -86,7 +86,7 @@ namespace Test.BuildXL.FrontEnd.Nuget var allPackages = new Dictionary { [packageOnDisk.Package.Id] = packageOnDisk.Package }; - var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false); + var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false); XAssert.IsTrue(analyzedPackage.Succeeded); @@ -105,10 +105,10 @@ namespace Test.BuildXL.FrontEnd.Nuget var packageText = File.ReadAllText(packageDsc); XAssert.IsFalse(packageText.Contains("export * from")); XAssert.IsFalse(packageText.Contains("/Foo.Bar.dsc\"")); - XAssert.IsTrue(packageText.Contains("/pkgs/Foo.Bar.1.2`")); - XAssert.IsTrue(packageText.Contains("packageRoot = d`")); - XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/a.txt`")); - XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/b.txt`")); + XAssert.IsTrue(packageText.Contains(@"id: ""Foo.Bar""")); + XAssert.IsTrue(packageText.Contains(@"version: ""1.2""")); + XAssert.IsTrue(packageText.Contains("r`Folder/a.txt`")); + XAssert.IsTrue(packageText.Contains("r`Folder/b.txt`")); } [Fact] @@ -140,7 +140,7 @@ namespace Test.BuildXL.FrontEnd.Nuget var pathTable = m_testContext.PathTable; var packageOnDisk = CreateTestPackageOnDisk(includeScriptSpec: false, version: version); var nugetResolver = CreateWorkspaceNugetResolverForTesting(); - var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false); + var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false); XAssert.IsTrue(analyzedPackage.Succeeded); var allPackages = new Dictionary { [packageOnDisk.Package.Id] = analyzedPackage.Result }; XAssert.IsTrue(analyzedPackage.Succeeded); @@ -314,7 +314,7 @@ $@"module({{ { var allPackages = packagesOnDisk.ToDictionary(kvp => kvp.Package.Id, kvp => kvp.Package); - var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, false).Result); + var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, AbsolutePath.Invalid, false).Result); resolver.SetDownloadedPackagesForTesting(allAnalyzedPackages); diff --git a/Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs b/Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs index 693a211f4..7da2b4a8c 100644 --- a/Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs +++ b/Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs @@ -44,7 +44,7 @@ namespace Test.BuildXL.FrontEnd.Nuget paths, "testPackageHash")); - return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false); + return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid); } public NugetAnalyzedPackage AnalyzePackageStub(Dictionary packagesOnConfig) @@ -58,7 +58,7 @@ namespace Test.BuildXL.FrontEnd.Nuget new PackageIdentity("nuget", nugetPackage.Id, nugetPackage.Version, nugetPackage.Alias), packageFolder: AbsolutePath.Create(m_context.PathTable, X("/X/Pkgs/TestPkgStub/1.999")))); - return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false); + return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid); } } } diff --git a/Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs b/Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs index f231231af..11a098e5c 100644 --- a/Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs +++ b/Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs @@ -86,18 +86,6 @@ namespace Test.DScript.Workspaces.Utilities { return new MalformedConfigurationFailure("An array literal is expected for NuGet credential providers"); } - - var credentialProviders = expression.Cast(); - foreach (var element in credentialProviders.Elements) - { - var elementResult = ParseNugetConfigurationFrom(element.As()); - if (!elementResult.Succeeded) - { - return elementResult; - } - - result.CredentialProviders.Add(elementResult.Result); - } } if (configurationExpression.TryFindAssignmentPropertyInitializer(NaiveConfigurationParsingUtilities.ToolUrlFieldName, out expression)) diff --git a/Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs b/Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs index c7d8524fc..735deb43f 100644 --- a/Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs +++ b/Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs @@ -145,11 +145,12 @@ namespace BuildXL.Ide.LanguageServer.UnitTests .OrderBy(ci => ci.Label) .ToArray(); - Assert.Equal(1, completionItems.Length); + Assert.Equal(2, completionItems.Length); // TODO: Change to just BuildXL.DScript check after April 15, 2019 when deployed bits have updated. Assert.True(completionItems[0].Label == "BuildXL.DScript.LanguageServer.UnitTests.Data.Module" || completionItems[0].Label == "BuildXLScript.LanguageServer.UnitTests.Data.Module"); + Assert.True(completionItems[1].Label == "BuildXL.Tools.NugetDownloader"); } [Fact] diff --git a/Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs b/Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs index b24318790..c8aa5e7ea 100644 --- a/Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs +++ b/Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs @@ -413,9 +413,9 @@ namespace BuildXL.Pips.Graph /// /// Add fields to fingerprint /// - public void AddFingerprint(IHashingHelper fingerprinter) + public void AddFingerprint(IHashingHelper fingerprinter, bool bypassFingerprintSaltAndVersion) { - if (!string.IsNullOrEmpty(FingerprintSalt)) + if (!bypassFingerprintSaltAndVersion && !string.IsNullOrEmpty(FingerprintSalt)) { fingerprinter.Add(nameof(FingerprintSalt), FingerprintSalt); } @@ -455,14 +455,17 @@ namespace BuildXL.Pips.Graph fingerprinter.Add(nameof(ExplicitlyReportDirectoryProbes), 1); } - fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion); + if (!bypassFingerprintSaltAndVersion) + { + fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion); + } } private CalculatedFingerprintTuple ComputeWeakFingerprint() { using (var hasher = new CoreHashingHelper(true)) { - AddFingerprint(hasher); + AddFingerprint(hasher, bypassFingerprintSaltAndVersion: false); return new CalculatedFingerprintTuple(hasher.GenerateHash(), hasher.FingerprintInputText); } } diff --git a/Public/Src/Pips/Dll/Graph/PipFingerprinter.cs b/Public/Src/Pips/Dll/Graph/PipFingerprinter.cs index 03a5d9eac..c3cb1882a 100644 --- a/Public/Src/Pips/Dll/Graph/PipFingerprinter.cs +++ b/Public/Src/Pips/Dll/Graph/PipFingerprinter.cs @@ -173,7 +173,7 @@ namespace BuildXL.Pips.Graph Contract.Requires(fingerprinter != null); Contract.Requires(pip != null); - fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp)); + fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp, pip.BypassFingerprintSalt)); // Fingerprints must change when outputs are hashed with a different algorithm. fingerprinter.Add(PipFingerprintField.ContentHashAlgorithmName, s_outputContentHashAlgorithmName); diff --git a/Public/Src/Pips/Dll/Operations/Pip.cs b/Public/Src/Pips/Dll/Operations/Pip.cs index e100616f1..4cf81c370 100644 --- a/Public/Src/Pips/Dll/Operations/Pip.cs +++ b/Public/Src/Pips/Dll/Operations/Pip.cs @@ -70,6 +70,11 @@ namespace BuildXL.Pips.Operations [PipCaching(FingerprintingRole = FingerprintingRole.None)] public abstract PipType PipType { get; } + /// + /// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both EngineEnvironmentSettings.DebugFingerprintSalt and PipFingerprintingVersion.TwoPhaseV2 + /// + public virtual bool BypassFingerprintSalt => false; + /// internal Pip() { diff --git a/Public/Src/Pips/Dll/Operations/Process.Options.cs b/Public/Src/Pips/Dll/Operations/Process.Options.cs index 6940e970c..8d4a1bab0 100644 --- a/Public/Src/Pips/Dll/Operations/Process.Options.cs +++ b/Public/Src/Pips/Dll/Operations/Process.Options.cs @@ -182,6 +182,11 @@ namespace BuildXL.Pips.Operations /// Whether to disable sandboxing for this process /// DisableSandboxing = 1 << 21, + + /// + /// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both and PipFingerprintingVersion.TwoPhaseV2 + /// + BypassFingerprintSalt = 1 << 22, } } } diff --git a/Public/Src/Pips/Dll/Operations/Process.cs b/Public/Src/Pips/Dll/Operations/Process.cs index 857d115ef..ff04e4caf 100644 --- a/Public/Src/Pips/Dll/Operations/Process.cs +++ b/Public/Src/Pips/Dll/Operations/Process.cs @@ -842,6 +842,10 @@ namespace BuildXL.Pips.Operations /// public bool HasSharedOpaqueDirectoryOutputs => DirectoryOutputs.Any(d => d.IsSharedOpaque); + /// + [PipCaching(FingerprintingRole = FingerprintingRole.Semantic)] + public override bool BypassFingerprintSalt => (ProcessOptions & Options.BypassFingerprintSalt) != 0; + /// /// What policy to apply when merging redirected outputs back /// diff --git a/Public/Src/Sandbox/MacOs/deployment.dsc b/Public/Src/Sandbox/MacOs/deployment.dsc index e23646b25..56d786c93 100644 --- a/Public/Src/Sandbox/MacOs/deployment.dsc +++ b/Public/Src/Sandbox/MacOs/deployment.dsc @@ -7,8 +7,6 @@ import * as SdkDeployment from "Sdk.Deployment"; namespace Deployment { export declare const qualifier: {configuration: "debug" | "release"}; - const pkgPath = d`${importFrom("runtime.osx-x64.BuildXL").Contents.all.root}/runtimes/osx-x64/native`; - @@public export const macBinaryUsage = Context.getCurrentHost().os === "macOS" ? (BuildXLSdk.Flags.isValidatingOsxRuntime ? "package" : "build") @@ -22,7 +20,9 @@ namespace Deployment { contents: macBinaryUsage === "none" ? [] : macBinaryUsage === "package" - ? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext`) ] + ? [ SdkDeployment.createFromFilteredStaticDirectory( + importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext`}), + r`.`)] : [{ subfolder: a`Contents`, contents: [ @@ -47,7 +47,9 @@ namespace Deployment { contents: macBinaryUsage === "none" ? [] : macBinaryUsage === "package" - ? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`) ] + ? [ SdkDeployment.createFromFilteredStaticDirectory( + importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`}), + r`.`)] : [{ subfolder: a`Contents`, contents: [ @@ -83,30 +85,34 @@ namespace Deployment { @@public export const sandboxMonitor: SdkDeployment.Definition = { - contents: [ + contents: macBinaryUsage === "build" - ? Sandbox.monitor - : f`${pkgPath}/${qualifier.configuration}/SandboxMonitor` - ] + ? [Sandbox.monitor] + : macBinaryUsage === "package" + ? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/SandboxMonitor`)] + : [] }; @@public export const interopLibrary: SdkDeployment.Definition = { contents: macBinaryUsage === "build" ? [ Sandbox.libInterop, Sandbox.libDetours ] - : [ - f`${pkgPath}/${qualifier.configuration}/libBuildXLInterop.dylib`, - f`${pkgPath}/${qualifier.configuration}/libBuildXLDetours.dylib` - ] + : macBinaryUsage === "package" + ? [ + importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLInterop.dylib`), + importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLDetours.dylib`) + ] + : [] }; @@public export const coreDumpTester: SdkDeployment.Definition = { - contents: [ + contents: macBinaryUsage === "build" - ? Sandbox.coreDumpTester - : f`${pkgPath}/${qualifier.configuration}/CoreDumpTester` - ] + ? [Sandbox.coreDumpTester] + : macBinaryUsage === "package" + ? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/CoreDumpTester`)] + : [] }; @@public diff --git a/Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs b/Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs index 6261dc80c..c6a44fc35 100644 --- a/Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs +++ b/Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs @@ -218,7 +218,7 @@ namespace BuildXL.FrontEnd.Script.Analyzer Contract.AssertNotNull(frontEndEngineAbstraction); AddConfigurationMounts(config, mountsTable); - languageServiceEngine.SetMountsTable(mountsTable); + languageServiceEngine.UpdateMountsTable(mountsTable); } if (frontEndEngineAbstraction == null) diff --git a/Public/Src/Tools/FileDownloader/Downloader.cs b/Public/Src/Tools/FileDownloader/Downloader.cs index 3917ce9a6..3aae6b4b4 100644 --- a/Public/Src/Tools/FileDownloader/Downloader.cs +++ b/Public/Src/Tools/FileDownloader/Downloader.cs @@ -2,21 +2,19 @@ // Licensed under the MIT License. using System; -using System.ComponentModel; using System.Diagnostics; using System.IO; -using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Text; +using System.Threading; using System.Threading.Tasks; using BuildXL.Cache.ContentStore.Hashing; using BuildXL.Native.IO; using BuildXL.Storage; using BuildXL.ToolSupport; using BuildXL.Utilities; -using Microsoft.IdentityModel.Clients.ActiveDirectory; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; +using BuildXL.Utilities.VstsAuthentication; namespace Tool.Download { @@ -94,11 +92,15 @@ namespace Tool.Download var httpRequest = new HttpRequestMessage(HttpMethod.Get, arguments.Url); + var logger = new StringBuilder(); + // If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request // We only want to send the token over HTTPS and to a VSTS domain to avoid security issues - if (IsVSTSPackageSecureURI(arguments.Url) && - await TryGetAuthenticationHeaderAsync(arguments.Url) is var authHeader && - authHeader != null) + if (VSTSAuthenticationHelper.IsVSTSPackageSecureURI(arguments.Url) && + await VSTSAuthenticationHelper.IsAuthenticationRequiredAsync(arguments.Url, CancellationToken.None, logger) && + await VSTSAuthenticationHelper.TryGetAuthenticationCredentialsAsync(arguments.Url, CancellationToken.None) is var maybeAuthCredentials && + maybeAuthCredentials.Succeeded && + VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(maybeAuthCredentials.Result.pat) is var authHeader) { httpRequest.Headers.Accept.Clear(); httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); @@ -162,175 +164,5 @@ namespace Tool.Download return await ContentHashingUtilities.HashContentStreamAsync(fs, hashType); } - - private bool IsVSTSPackageSecureURI(Uri downloadURI) - { - return downloadURI.Scheme == "https" && - (downloadURI.Host.EndsWith(".pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith(".pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase)) ; - } - - /// - /// Tries to authenticate using a credential provider and fallback to IWA if that fails. - /// - private async Task TryGetAuthenticationHeaderAsync(Uri uri) - { - var result = await TryGetAuthenticationHeaderValueWithCredentialProviderAsync(uri); - if (result != null) - { - return result; - } - - return await TryGetAuthenticationHeaderWithIWAAsync(uri); - } - - /// - /// Tries to get an authentication token using Integrated Windows Authentication with the current logged in user - /// - /// Null if authentication fails - /// - /// The auth token is acquired to address simple auth cases for retrieving packages from VSTS feeds from Windows domain - /// joined machines where IWA is enabled. - /// See https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/wiki/Integrated-Windows-Authentication - /// - private async Task TryGetAuthenticationHeaderWithIWAAsync(Uri uri) - { - var authenticationContext = new AuthenticationContext(m_authority); - - try - { - var userCredential = new UserCredential($"{Environment.UserName}@{Tenant}"); - - // Many times the user UPN cannot be automatically retrieved, so we build it based on the current username and tenant. This might - // not be perfect but should work for most cases. Getting the UPN for a given user from AD requires authentication as well. - var result = await authenticationContext.AcquireTokenAsync(Resource, Client, userCredential); ; - return new AuthenticationHeaderValue("Bearer", result.AccessToken); - } - catch (AdalException ex) - { - Console.WriteLine($"Download resolver was not able to authenticate using Integrated Windows Authentication for '{uri}': {ex.Message}"); - // Getting an auth token via IWA is on a best effort basis. If anything fails, we silently continue without auth - return null; - } - } - - /// - /// Tries to get an authentication token using a credential provider - /// - /// Null if authentication fails - /// - /// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers - /// using an environment variable. - /// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general - /// - private async Task TryGetAuthenticationHeaderValueWithCredentialProviderAsync(Uri uri) - { - string credentialProviderPath = DiscoverCredentialProvider(uri); - - if (credentialProviderPath == null) - { - // Failure has been logged already - return null; - } - - // Call the provider with the requested URI in non-interactive mode. - var processInfo = new ProcessStartInfo - { - FileName = credentialProviderPath, - Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive", - RedirectStandardOutput = true, - CreateNoWindow = true, - }; - - using (var process = Process.Start(processInfo)) - { - try - { - #pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead - process?.WaitForExit(); - #pragma warning restore AsyncFixer02 - } - catch (Exception e) when (e is SystemException || e is Win32Exception) - { - ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}"); - return null; - } - - if (process == null) - { - // The process was not started - ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not start credential provider process '{credentialProviderPath}'."); - return null; - } - - // Check whether the authorization succeeded - if (process.ExitCode == 0) - { - try - { - // The response should be a well-formed JSON with a 'password' entry representing the PAT - var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput)); - var pat = response.GetValue("Password"); - - if (pat == null) - { - ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'."); - return null; - } - - // We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT - return new AuthenticationHeaderValue("Basic", - Convert.ToBase64String( - System.Text.ASCIIEncoding.ASCII.GetBytes( - string.Format("{0}:{1}", "", pat)))); - } - catch (JsonReaderException) - { - ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' response is not a well-formed JSON."); - return null; - } - } - - ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}."); - } - - return null; - } - - private string DiscoverCredentialProvider(Uri uri) - { - var paths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH"); - - if (paths == null) - { - ReportAuthenticationViaCredentialProviderFailed(uri, $"NUGET_CREDENTIALPROVIDERS_PATH is not set."); - return null; - } - - // Here we do something slightly simpler than what NuGet does and just look for the first credential - // provider we can find - string credentialProviderPath = null; - foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) - { - credentialProviderPath = Directory.EnumerateFiles(path, "credentialprovider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault(); - if (credentialProviderPath != null) - { - break; - } - } - - if (credentialProviderPath == null) - { - ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider was not found under '{paths}'."); - - return null; - } - - return credentialProviderPath; - } - - private void ReportAuthenticationViaCredentialProviderFailed(Uri url, string details) - { - Console.WriteLine($"Download resolver was not able to authenticate using a credential provider for '{url}': {details}") ; - } } } diff --git a/Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc b/Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc index 20ef01191..ca401cb02 100644 --- a/Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc +++ b/Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc @@ -21,6 +21,7 @@ namespace FileDownloader { sources: globR(d`.`, "Downloader*.cs"), references:[ importFrom("Newtonsoft.Json").pkg, + importFrom("BuildXL.Utilities").VstsAuthentication.dll, importFrom("BuildXL.Utilities").Collections.dll, importFrom("BuildXL.Utilities").Native.dll, importFrom("BuildXL.Utilities.Instrumentation").Common.dll, diff --git a/Public/Src/Tools/NugetDownloader/App.config b/Public/Src/Tools/NugetDownloader/App.config new file mode 100644 index 000000000..cd04a1cba --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/App.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/Public/Src/Tools/NugetDownloader/ConsoleLogger.cs b/Public/Src/Tools/NugetDownloader/ConsoleLogger.cs new file mode 100644 index 000000000..6d79a7323 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/ConsoleLogger.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Threading.Tasks; +using NuGet.Common; + +namespace NugetDownloader +{ + /// + /// Logs all messages to the console + /// + internal class ConsoleLogger : ILogger + { + /// + public void Log(LogLevel level, string data) + { + Console.WriteLine($"[{level}]:{data}"); + } + + /// + public void Log(ILogMessage message) + { + Console.WriteLine(message.FormatWithCode()); + } + + /// + public Task LogAsync(LogLevel level, string data) + { + Log(level, data); + return Task.CompletedTask; + } + + /// + public Task LogAsync(ILogMessage message) + { + Log(message); + return Task.CompletedTask; + + } + + /// + public void LogDebug(string data) + { + Log(LogLevel.Debug, data); + } + + /// + public void LogError(string data) + { + Log(LogLevel.Error, data); + } + + /// + public void LogInformation(string data) + { + Log(LogLevel.Information, data); + } + + /// + public void LogInformationSummary(string data) + { + Log(LogLevel.Information, data); + } + + /// + public void LogMinimal(string data) + { + Log(LogLevel.Information, data); + } + + /// + public void LogVerbose(string data) + { + Log(LogLevel.Verbose, data); + } + + /// + public void LogWarning(string data) + { + Log(LogLevel.Warning, data); + } + } +} diff --git a/Public/Src/Tools/NugetDownloader/LiteralFiles/Tool.NugetDownloader.dsc.literal b/Public/Src/Tools/NugetDownloader/LiteralFiles/Tool.NugetDownloader.dsc.literal new file mode 100644 index 000000000..a1656a314 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/LiteralFiles/Tool.NugetDownloader.dsc.literal @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import {Artifact, Cmd, Transformer, Tool} from "Sdk.Transformers"; + +export declare const qualifier: {}; + +const downloaderDefinition: Transformer.ToolDefinition = { + // CODESYNC: keep in sync with deployment at Public\Src\FrontEnd\Nuget\BuildXL.FrontEnd.Nuget.dsc + exe: Context.isWindowsOS()? f`${Context.getBuildEngineDirectory()}/NugetDownloader.exe` : f`${Context.getBuildEngineDirectory()}/NugetDownloader`, + description: "BuildXL NuGet downloader", + dependsOnWindowsDirectories: true, + dependsOnAppDataDirectory: true, + prepareTempDirectory: true, + untrackedDirectoryScopes: [ + d`${Context.getMount("ProgramData").path}/Microsoft/NetFramework/BreadcrumbStore`, + ...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("LocalLow").path}/Microsoft/CryptnetUrlCache`]), + ...addIfLazy(Context.getCurrentHost().os === "macOS", () => [d`/dev/dtracehelper`, d`/private/var/run/mDNSResponder`, d`/private/var/folders`, d`/dev/tty`, d`/usr/share/zoneinfo`, d`/usr/share/icu`]), + d`${Context.getMount("UserProfile").path}/.dotnet/corefx/cryptography` + ], +}; + + +/** + * Arguments for downloading a NuGet package under a specified directory + */ +@@public +export interface Arguments extends Transformer.RunnerArguments { + id: string; + version: string; + downloadDirectory: Directory; + extractedFiles: RelativePath[]; + repositories: [string, string][]; + credentialProviderPath?: File; + timeoutInMinutes?: number; +} + +/** + * Downloads a NuGet package as specified in the arguments and returns the outputs that result of the execution in a static directory + */ +@@public +export function downloadPackage(args: Arguments) : PartialStaticContentDirectory { + + const arguments: Argument[] = [ + Cmd.option("/id:", args.id), + Cmd.option("/version:", args.version), + Cmd.option("/downloadDirectory:", Artifact.none(args.downloadDirectory)), + Cmd.options("/repositories:", args.repositories.map(kvp => kvp[0] + "=" + kvp[1])), + ]; + + // NuGet extraction always injects a parent folder with the package id + version as the name + const directoryName = `${args.id}.${args.version}`; + + // Build the list of expected outputs + const outputs : File[] = args.extractedFiles.map(relativePath => f`${args.downloadDirectory}/${directoryName}/${relativePath}`); + + // In some cases the package is expected to be empty. In that case, don't run anything and just return an empty seal dir + if (outputs.length === 0) + { + return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, []); + } + + let tool = downloaderDefinition; + + // The timeout is not available per-pip but at the tool level + if (args.timeoutInMinutes !== undefined) { + tool = tool.merge({ + timeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000, + // We could have a separate argument for this, but for now let's keep it simple + warningTimeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000 + }); + } + + // Nuget tries to read the user settings config file, and if it is not there, it creates it. + // The creation does not behave well with concurrent executions and write locks issues arise. + // Let's give each nuget pip a different app data directory + const redirectedAppData = Context.getTempDirectory("redirectedAppData"); + + const transformerExecuteArgs : Transformer.ExecuteArguments = { + description: `Downloading NuGet package '${args.id}.${args.version}'`, + tool: tool, + arguments: arguments, + workingDirectory: args.downloadDirectory, + tags: ["nuget", ...(args.tags || [])], + environmentVariables: [ + ...addIf(args.credentialProviderPath !== undefined, {name: "NUGET_CREDENTIALPROVIDERS_PATH", value: args.credentialProviderPath}), + ...addIf(Environment.hasVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS"), {name: "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", value: Environment.getStringValue("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS")}), + // In case the artifacts credential provider is being used, make sure we don't cache the token, so we avoid dealing with IsRetry + {name: "NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED", value: "False"}, + // On linux/mac the NuGet SDK needs this variable defined. It doesn't really matter where it is pointing to. + ...addIf(!Context.isWindowsOS(), {name: "DOTNET_CLI_HOME", value: redirectedAppData}), + {name: "AppData", value: redirectedAppData} + ], + outputs: outputs, + tempDirectory: redirectedAppData, + unsafe: { + passThroughEnvironmentVariables: [ + "LocalAppData", + "QAUTHMATERIALROOT", + "__CREDENTIAL_PROVIDER_LOG_DIR", + "__CLOUDBUILD_AUTH_HELPER_ROOT__", + "__CLOUDBUILD_AUTH_HELPER_CONFIG__", + ], + requireGlobalDependencies: false, + // The only child process this tool spawns is the credential provider process. This process may write log files that are hard to predict upfront. + hasUntrackedChildProcesses: true, + // This is to avoid a cache miss on a NuGet download due to changes on the nuget download tool (changes in downtream dependencies for the + // most part, e.g. some framework dll that is changed because of some unrelated change in Bxl) + // This is a confidence vote since we own the nuget downloader tool. If for any reason the nuget pips need invalidation, that can + // be manually triggered by bumping the NuGet spec fingerprint version. + untrackedScopes: [Context.getBuildEngineDirectory()], + // Make NuGet pips so fingerprint salts (the general fingerprint V2 version and the debug salt passed with /p:BUILDXL_FINGERPRINT_SALT) are not + // included in the weak fingerprint. Observe this option is not exposed in DScript, so we are forcing the hand of the type checker with the cast above + // to not complain about this extra field (which will be picked up at runtime) + bypassFingerprintSalt: true + }, + // This is to avoid DFAs under the credential provider directory, which is an arbitrarily specified directory that would need to + // be contained in a statically declared read mount otherwise + allowUndeclaredSourceReads: true, + }; + + const result = Transformer.execute(transformerExecuteArgs); + + return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, result.getOutputFiles()); +} \ No newline at end of file diff --git a/Public/Src/Tools/NugetDownloader/LiteralFiles/module.config.dsc.literal b/Public/Src/Tools/NugetDownloader/LiteralFiles/module.config.dsc.literal new file mode 100644 index 000000000..96fcccb69 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/LiteralFiles/module.config.dsc.literal @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +module({ + name: "BuildXL.Tools.NugetDownloader" +}); diff --git a/Public/Src/Tools/NugetDownloader/NugetDownloader.cs b/Public/Src/Tools/NugetDownloader/NugetDownloader.cs new file mode 100644 index 000000000..e513584a6 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/NugetDownloader.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using BuildXL.Native.IO; +using BuildXL.ToolSupport; +using BuildXL.Utilities; +using BuildXL.Utilities.VstsAuthentication; +using NuGet.Common; +using NuGet.Configuration; +using NuGet.Packaging; +using NuGet.Packaging.Signing; +using NuGet.Protocol.Core.Types; +using NugetDownloader; + +namespace Tool.Download +{ + /// + /// Downloads a Nuget into a given directory from a collection of source repositories + /// + internal sealed class NugetDownloader : ToolProgram + { + private NugetDownloader() : base("NugetDownloader") + { + } + + /// + public static int Main(string[] arguments) + { + return new NugetDownloader().MainHandler(arguments); + } + + /// + public override bool TryParse(string[] rawArgs, out NugetDownloaderArgs arguments) + { + try + { + arguments = new NugetDownloaderArgs(rawArgs); + return true; + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.GetLogEventMessage()); + arguments = null; + return false; + } + } + + /// + public override int Run(NugetDownloaderArgs arguments) + { + return TryDownloadNugetToDiskAsync(arguments).GetAwaiter().GetResult(); + } + + /// + /// Attempts to download and extract a NuGet package to disk. + /// + private async Task TryDownloadNugetToDiskAsync(NugetDownloaderArgs arguments) + { + Console.WriteLine($"Download started for package '{arguments.Id}' and version '{arguments.Version}' to '{arguments.DownloadDirectory}'."); + + var stopwatch = Stopwatch.StartNew(); + + SourceCacheContext cache = new SourceCacheContext(); + + var authLogger = new StringBuilder(); + var maybeRepositories = await VSTSAuthenticationHelper.TryCreateSourceRepositories(arguments.Repositories.Select(kvp => (kvp.Key, kvp.Value)), CancellationToken.None, authLogger); + + // Informational data + Console.WriteLine(authLogger.ToString()); + + if (!maybeRepositories.Succeeded) + { + Console.Error.WriteLine($"Failed to access the specified repositories: " + maybeRepositories.Failure.Describe()); + return 1; + } + + bool found = false; + ILogger logger = new ConsoleLogger(); + foreach (var sourceRepository in maybeRepositories.Result) + { + Console.Write($"Finding resource against source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}'. Authentication is on: {sourceRepository.PackageSource.Credentials != null}"); + + // TODO: maybe we need a retry here? or define a default retry in the DScript SDK? + FindPackageByIdResource resource = await sourceRepository.GetResourceAsync(); + + if (resource == null) + { + Console.Write($"Resource under source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}' not found."); + continue; + } + + var destinationFilePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + + // We don't really need to materialize the nupkg file on disk, but a memory stream has a 2GB limitation, and NuGet packages + // can be larger than that. Just place it on disk and remove it after we are done extracting. + using (var packageStream = new FileStream( + destinationFilePath, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.ReadWrite | FileShare.Delete, + bufferSize: 4096)) + { + if (await resource.CopyNupkgToStreamAsync( + arguments.Id, + arguments.Version, + packageStream, + cache, + logger, + CancellationToken.None)) + { + found = true; + + await PackageExtractor.ExtractPackageAsync( + sourceRepository.PackageSource.Source, + packageStream, + new PackagePathResolver(arguments.DownloadDirectory), + new PackageExtractionContext( + PackageSaveMode.Files | PackageSaveMode.Nuspec, + XmlDocFileSaveMode.None, + ClientPolicyContext.GetClientPolicy(NullSettings.Instance, logger), + logger), + CancellationToken.None); + + FileUtilities.DeleteFile(destinationFilePath); + + break; + } + } + } + + if (!found) + { + Console.Error.WriteLine($"Could not find NuGet package '{arguments.Id}' with version '{arguments.Version.ToNormalizedString()}' under any of the provided repositories."); + return 1; + } + + Console.WriteLine($"Finished downloading package '{arguments.Id}' with version '{arguments.Version}' in {stopwatch.ElapsedMilliseconds}ms"); + + return 0; + } + } +} diff --git a/Public/Src/Tools/NugetDownloader/NugetDownloaderArgs.cs b/Public/Src/Tools/NugetDownloader/NugetDownloaderArgs.cs new file mode 100644 index 000000000..57a01ea32 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/NugetDownloaderArgs.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using BuildXL.ToolSupport; +using NuGet.Versioning; + +namespace Tool.Download +{ + /// + internal sealed partial class NugetDownloaderArgs : CommandLineUtilities + { + /// + /// package id of the download + /// + public string Id { get; } + + /// + /// package version of the download + /// + public NuGetVersion Version { get; } + + /// + /// The directory to place the downloaded file + /// + public string DownloadDirectory { get; } + + /// + /// The Nuget feeds (name and Uri) where to download the package from + /// + public IReadOnlyDictionary Repositories { get; } + + /// + public NugetDownloaderArgs(string[] args) + : base(args) + { + var repositories = new Dictionary(); + + foreach (Option opt in Options) + { + if (opt.Name.Equals("id", StringComparison.OrdinalIgnoreCase)) + { + Id = opt.Value; + } + else if (opt.Name.Equals("version", StringComparison.OrdinalIgnoreCase)) + { + if (!NuGetVersion.TryParse(opt.Value, out NuGetVersion version)) + { + throw Error($"Malformed version: {opt.Value}."); + } + + Version = version; + } + else if (opt.Name.Equals("downloadDirectory", StringComparison.OrdinalIgnoreCase)) + { + DownloadDirectory = opt.Value; + } + else if (opt.Name.Equals("repositories", StringComparison.OrdinalIgnoreCase)) + { + var kvp = ParseKeyValuePair(opt); + repositories[kvp.Key] = new Uri(kvp.Value); + } + else + { + throw Error($"Unsupported option: {opt.Name}."); + } + } + + Repositories = repositories; + + if (Id == null) + { + throw Error($"Missing mandatory argument 'id'"); + } + + if (Version == null) + { + throw Error($"Missing mandatory argument 'version'"); + } + + if (Repositories.Count == 0) + { + throw Error($"Missing mandatory argument 'repositories'"); + } + + if (string.IsNullOrWhiteSpace(DownloadDirectory)) + { + throw Error($"Missing mandatory argument 'DownloadDirectory'"); + } + } + } +} diff --git a/Public/Src/Tools/NugetDownloader/Tool.NugetDownloader.dsc b/Public/Src/Tools/NugetDownloader/Tool.NugetDownloader.dsc new file mode 100644 index 000000000..3944d54f5 --- /dev/null +++ b/Public/Src/Tools/NugetDownloader/Tool.NugetDownloader.dsc @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +import * as Managed from "Sdk.Managed"; +import * as BuildXLSdk from "Sdk.BuildXL"; +import {Transformer} from "Sdk.Transformers"; +import * as Deployment from "Sdk.Deployment"; +import * as MSBuild from "Sdk.Selfhost.MSBuild"; +import * as Frameworks from "Sdk.Managed.Frameworks"; +import * as Shared from "Sdk.Managed.Shared"; + +namespace NugetDownloader { + + export declare const qualifier : BuildXLSdk.DefaultQualifierWithNet472; + + // These are the specs that define the nuget downloader SDK. Since this is a bxl-provided tool + // and customers don't have direct exposure to it, they are not places under the usual SDK folder, but + // deployed as part of bxl binaries + @@public + export const specDeployment: Deployment.Definition = { + contents: [ + {subfolder: r`Sdk/Sdk.NugetDownloader`, contents: [ + {file: f`LiteralFiles/Tool.NugetDownloader.dsc.literal`, targetFileName: a`Tool.NugetDownloader.dsc`}, + {file: f`LiteralFiles/module.config.dsc.literal`, targetFileName: a`module.config.dsc`},] + } + ] + }; + + @@public + export const dll = BuildXLSdk.executable({ + assemblyName: "NugetDownloader", + skipDocumentationGeneration: true, + skipDefaultReferences: true, + sources: globR(d`.`, "*.cs"), + references:[ + ...addIf(BuildXLSdk.isFullFramework, + NetFx.Netstandard.dll + ), + importFrom("BuildXL.Utilities").VstsAuthentication.dll, + importFrom("BuildXL.Utilities").ToolSupport.dll, + importFrom("BuildXL.Utilities").Native.dll, + importFrom("BuildXL.Utilities").dll, + + importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg, + ], + runtimeContentToSkip:[ + importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg, + ], + runtimeContent: [ + f`App.config`, + specDeployment + ], + deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Collections.Immutable.dll`] }, + }); +} + + diff --git a/Public/Src/Utilities/Configuration/FrontEndConfigurationExtensions.cs b/Public/Src/Utilities/Configuration/FrontEndConfigurationExtensions.cs index 323217081..017b00fe7 100644 --- a/Public/Src/Utilities/Configuration/FrontEndConfigurationExtensions.cs +++ b/Public/Src/Utilities/Configuration/FrontEndConfigurationExtensions.cs @@ -33,7 +33,7 @@ namespace BuildXL.Utilities.Configuration public const bool DefaultForcePopulatePackageCache = false; /// - public const bool DefaultRespectWeakFingerprint = false; + public const bool DefaultRespectWeakFingerprint = true; /// public const bool DefaultForceGenerateNuGetSpecs = false; diff --git a/Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs b/Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs index 93dcf23c0..33ec05c37 100644 --- a/Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs +++ b/Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs @@ -11,8 +11,8 @@ namespace BuildXL.Utilities.Configuration public partial interface INugetConfiguration : IArtifactLocation { /// - /// Optional credential helper to use for NuGet + /// The download timeout, in minutes, for each NuGet download pip /// - IReadOnlyList CredentialProviders { get; } + int? DownloadTimeoutMin { get; } } } diff --git a/Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs b/Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs index b329aacad..ede9c39aa 100644 --- a/Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs +++ b/Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs @@ -16,6 +16,7 @@ namespace BuildXL.Utilities.Configuration.Mutable Repositories = new Dictionary(); Packages = new List(); DoNotEnforceDependencyVersions = false; + Configuration = new NugetConfiguration(); } /// @@ -25,7 +26,7 @@ namespace BuildXL.Utilities.Configuration.Mutable Contract.Assume(template != null); Contract.Assume(pathRemapper != null); - Configuration = template.Configuration == null ? null : new NugetConfiguration(template.Configuration); + Configuration = new NugetConfiguration(template.Configuration); Repositories = new Dictionary(template.Repositories.Count); foreach (var kv in template.Repositories) { diff --git a/Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs b/Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs index 4de7ab5b6..7a1b1559d 100644 --- a/Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs +++ b/Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; - namespace BuildXL.Utilities.Configuration.Mutable { /// @@ -12,21 +9,17 @@ namespace BuildXL.Utilities.Configuration.Mutable /// public NugetConfiguration() { - CredentialProviders = new List(); + DownloadTimeoutMin = 20; } /// public NugetConfiguration(INugetConfiguration template) : base(template) { - CredentialProviders = new List(template.CredentialProviders); + DownloadTimeoutMin = template.DownloadTimeoutMin; } - /// - [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")] - public List CredentialProviders { get; set; } - - /// - IReadOnlyList INugetConfiguration.CredentialProviders => CredentialProviders; + /// + public int? DownloadTimeoutMin { get; set; } } } diff --git a/Public/Src/Utilities/Native/BuildXL.Native.dsc b/Public/Src/Utilities/Native/BuildXL.Native.dsc index acaff8cd2..22d11ca27 100644 --- a/Public/Src/Utilities/Native/BuildXL.Native.dsc +++ b/Public/Src/Utilities/Native/BuildXL.Native.dsc @@ -51,7 +51,7 @@ namespace Native { export const nativeLinux = [ ...addIfLazy(qualifier.targetRuntime === "linux-x64", () => [ - ...globR(d`${importFrom("runtime.linux-x64.BuildXL").Contents.all.root}/runtimes/linux-x64/native/${qualifier.configuration}`, "*.so") + ...importFrom("runtime.linux-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/linux-x64/native/${qualifier.configuration}`}).getContent() ]), ]; diff --git a/Public/Src/Utilities/UnitTests/Utilities/ZeroLeftPaddedStreamTests.cs b/Public/Src/Utilities/UnitTests/Utilities/ZeroLeftPaddedStreamTests.cs new file mode 100644 index 000000000..98c7e7f78 --- /dev/null +++ b/Public/Src/Utilities/UnitTests/Utilities/ZeroLeftPaddedStreamTests.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.IO; +using BuildXL.Utilities; +using Test.BuildXL.TestUtilities.Xunit; +using Xunit; +using Xunit.Abstractions; + +namespace Test.BuildXL.Utilities +{ + public class ZeroLeftPaddedStreamTests : XunitBuildXLTest + { + public ZeroLeftPaddedStreamTests(ITestOutputHelper output) + : base(output) { } + + [Fact] + public void TestBasicFunctionality() + { + var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 }); + var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10); + + // Total length should be 10, even if the underlying stream is smaller + XAssert.AreEqual(10, paddedStream.Length); + + // Check Position works + paddedStream.Position = 9; + XAssert.AreEqual(9, paddedStream.Position); + + // Read the whole stream. We should get zeros in the first 5 positions + paddedStream.Position = 0; + XAssert.AreEqual(0, paddedStream.Position); + var result = new byte[10]; + var bytesRead = paddedStream.Read(result, 0, 10); + XAssert.AreArraysEqual(new byte[] { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5 }, result, true); + XAssert.AreEqual(10, bytesRead); + + // Read a part of the stream offsetting the destination + System.Array.Clear(result, 0, 10); + paddedStream.Position = 4; + bytesRead = paddedStream.Read(result, 1, 3); + XAssert.AreArraysEqual(new byte[] { 0, 0, 1, 2, 0, 0, 0, 0, 0, 0 }, result, true); + XAssert.AreEqual(3, bytesRead); + } + + [Fact] + public void TestSeek() + { + var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 }); + var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10); + + var result = new byte[2]; + + var newPos = paddedStream.Seek(4, SeekOrigin.Begin); + XAssert.AreEqual(4, newPos); + paddedStream.Read(result, 0, 2); + XAssert.AreEqual(0, result[0]); + XAssert.AreEqual(1, result[1]); + + newPos = paddedStream.Seek(7, SeekOrigin.Begin); + XAssert.AreEqual(7, newPos); + paddedStream.Read(result, 0, 2); + XAssert.AreEqual(3, result[0]); + XAssert.AreEqual(4, result[1]); + + newPos = paddedStream.Seek(-1, SeekOrigin.End); + XAssert.AreEqual(9, newPos); + paddedStream.Read(result, 0, 1); + XAssert.AreEqual(5, result[0]); + + newPos = paddedStream.Seek(-6, SeekOrigin.End); + XAssert.AreEqual(4, newPos); + paddedStream.Read(result, 0, 2); + XAssert.AreEqual(0, result[0]); + XAssert.AreEqual(1, result[1]); + + paddedStream.Position = 6; + newPos = paddedStream.Seek(-2, SeekOrigin.Current); + XAssert.AreEqual(4, newPos); + paddedStream.Read(result, 0, 2); + XAssert.AreEqual(0, result[0]); + XAssert.AreEqual(1, result[1]); + + paddedStream.Position = 4; + newPos = paddedStream.Seek(2, SeekOrigin.Current); + XAssert.AreEqual(6, newPos); + paddedStream.Read(result, 0, 2); + XAssert.AreEqual(2, result[0]); + XAssert.AreEqual(3, result[1]); + } + } +} + diff --git a/Public/Src/Utilities/Utilities/ZeroLeftPaddedStream.cs b/Public/Src/Utilities/Utilities/ZeroLeftPaddedStream.cs new file mode 100644 index 000000000..40e91baef --- /dev/null +++ b/Public/Src/Utilities/Utilities/ZeroLeftPaddedStream.cs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Diagnostics.ContractsLight; +using System.IO; + +namespace BuildXL.Utilities +{ + /// + /// Wraps a seekable stream to expose it with a configurable minimum size, left padding with zeros if needed. + /// + /// + /// No extra memory is taken by the padded portion of the stream + /// If the underlying stream is modified, then the behavior of this class is non-deterministic + /// + public class ZeroLeftPaddedStream : Stream + { + private readonly Stream m_stream; + private long m_currentPos = 0; + private readonly long m_gap; + + /// + public ZeroLeftPaddedStream(Stream stream, long minimumLength = 0) + { + Contract.RequiresNotNull(stream); + Contract.Requires(stream.CanSeek); + + m_stream = stream; + MinimumLength = minimumLength; + m_gap = Math.Max(0, MinimumLength - m_stream.Length); + } + + /// + /// The miminum legth the stream has + /// + public long MinimumLength { set; get; } + + /// + public override bool CanRead => m_stream.CanRead; + + /// + public override bool CanSeek => m_stream.CanSeek; + + /// + /// Always false + /// + public override bool CanWrite => false; + + /// + public override long Length => m_stream.Length + m_gap; + + /// + public override long Position { + get => m_currentPos; + set + { + m_stream.Position = Math.Max(0, value - m_gap); + m_currentPos = Math.Min(value, Length); + } + } + + /// + public override void Flush() + { + m_stream.Flush(); + } + + /// + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = 0; + + if (Position < m_gap) + { + // Produce the padded area of the stream with 0-bytes + long maxPaddedCount = m_gap - Position; + int paddedCount = Math.Min(count, maxPaddedCount > int.MaxValue ? int.MaxValue : (int) maxPaddedCount); + if (paddedCount > 0) + { + Array.Clear(buffer, offset, paddedCount); + bytesRead = paddedCount; + } + + // Read the reminder, if any, from the underlying stream + if (paddedCount < count) + { + bytesRead += m_stream.Read(buffer, offset + paddedCount, count - paddedCount); + } + } + else + { + // The position is beyond the padded area, just do a regular read on the underlying stream + bytesRead = m_stream.Read(buffer, offset, count); + } + + Position += count; + + return bytesRead; + } + + /// + public override long Seek(long offset, SeekOrigin origin) + { + // Compute the origin of the seek in terms of the starting index on the stream + long originIndex = origin switch + { + SeekOrigin.Begin => 0, + SeekOrigin.Current => Position, + SeekOrigin.End => Length, + _ => throw new ArgumentOutOfRangeException(nameof(origin), $"Not expected value {origin}") + }; + + // This is the final index of the stream + long finalIndex = Math.Min(Length, originIndex + offset); + + if (finalIndex < 0) + { + throw new ArgumentException("Attempt to seek before the beginning of the stream"); + } + + if (finalIndex >= m_gap) + { + m_stream.Seek(finalIndex - m_gap, SeekOrigin.Begin); + } + else + { + m_stream.Seek(0, SeekOrigin.Begin); + } + + Position = finalIndex; + + return Position; + } + + /// + /// Always throws + /// + /// + /// Shouldn't be called since this stream is non-writable + /// + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + /// + /// Always throws + /// + /// + /// Shouldn't be called since this stream is non-writable + /// + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + } +} diff --git a/Public/Src/Utilities/VstsAuthentication/AuthenticationFailure.cs b/Public/Src/Utilities/VstsAuthentication/AuthenticationFailure.cs new file mode 100644 index 000000000..8dd0388ad --- /dev/null +++ b/Public/Src/Utilities/VstsAuthentication/AuthenticationFailure.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace BuildXL.Utilities.VstsAuthentication +{ + /// + /// An authentication related failure when dealing with authenticated NuGet feeds + /// + public class AuthenticationFailure : Failure + { + private readonly string m_failure; + + /// + public AuthenticationFailure(string failure) + { + m_failure = failure; + } + + /// + public override BuildXLException CreateException() + { + return new BuildXLException(m_failure); + } + + /// + public override string Describe() + { + return m_failure; + } + + /// + public override BuildXLException Throw() + { + throw CreateException(); + } + } +} diff --git a/Public/Src/Utilities/VstsAuthentication/BuildXL.Utilties.VstsAuthentication.dsc b/Public/Src/Utilities/VstsAuthentication/BuildXL.Utilties.VstsAuthentication.dsc new file mode 100644 index 000000000..6238cf47a --- /dev/null +++ b/Public/Src/Utilities/VstsAuthentication/BuildXL.Utilties.VstsAuthentication.dsc @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace VstsAuthentication { + @@public + export const dll = BuildXLSdk.library({ + assemblyName: "BuildXL.Utilities.VstsAuthentication", + sources: globR(d`.`, "*.cs"), + references: [ + $.dll, + Native.dll, + importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg, + importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg, + ...addIfLazy(BuildXLSdk.isFullFramework, () => [ + NetFx.System.Net.Http.dll, + NetFx.Netstandard.dll, + ]), + ], + }); +} \ No newline at end of file diff --git a/Public/Src/Utilities/VstsAuthentication/VSTSAuthenticationHelper.cs b/Public/Src/Utilities/VstsAuthentication/VSTSAuthenticationHelper.cs new file mode 100644 index 000000000..15ff82305 --- /dev/null +++ b/Public/Src/Utilities/VstsAuthentication/VSTSAuthenticationHelper.cs @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using System.Diagnostics.ContractsLight; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Configuration; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using System.Net.Http; +using BuildXL.Native.IO; +using System.Text; + +namespace BuildXL.Utilities.VstsAuthentication +{ + /// + /// Helper to get authentication credentials against VSTS connections + /// + public static class VSTSAuthenticationHelper + { + // Constant value to target Azure DevOps. + private const string Resource = "499b84ac-1321-427f-aa17-267ca6975798"; + // Visual Studio IDE client ID originally provisioned by Azure Tools. + private const string Client = "872cd9fa-d31f-45e0-9eab-6e460a02d1f1"; + // Microsoft tenant + private const string Tenant = "microsoft.com"; + // Microsoft authority + private const string Authority = $"https://login.windows.net/microsoft.com"; + + /// + /// Creates a collection of given a collection of URIs + /// + /// + /// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable + /// + public static Task>> TryCreateSourceRepositories( + IEnumerable<(string repositoryName, Uri repositoryUri)> repositories, + CancellationToken cancellationToken, + StringBuilder logger) + { + return TryCreateSourceRepositories(repositories, TryDiscoverCredentialProvider, cancellationToken, logger); + } + + /// + /// Creates a collection of given a collection of URIs + /// + /// + /// For the repositories identified as VSTS ones, credential provider authentication is performed + /// + public static async Task>> TryCreateSourceRepositories( + IEnumerable<(string repositoryName, Uri repositoryUri)> repositories, + Func> discoverCredentialProvider, + CancellationToken cancellationToken, + StringBuilder logger) + { + Contract.RequiresNotNull(repositories); + + var sourceRepositories = new List(); + + Lazy> credentialProviderPath = new Lazy>(discoverCredentialProvider); + + // For each repository, authenticate if we are dealing with a VSTS feed and retrieve the base addresses + foreach (var repository in repositories) + { + Uri uri = repository.repositoryUri; + string uriAsString = uri.AbsoluteUri; + var packageSource = new PackageSource(uriAsString, repository.repositoryName); + + logger.AppendLine($"Analyzing {uri.AbsoluteUri}"); + // For now we only support authentication against VSTS feeds. Feeds outside of that will attempt to connect unauthenticated + if (IsVSTSPackageSecureURI(uri) && await IsAuthenticationRequiredAsync(uri, cancellationToken, logger)) + { + logger.AppendLine($"Authentication required for {uri.AbsoluteUri}. Trying to acquire credentials."); + + if (!credentialProviderPath.Value.Succeeded) + { + return credentialProviderPath.Value.Failure; + } + + if (await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath.Value.Result, cancellationToken) is var maybeCredentials) + { + var maybePackageSourceCredential = maybeCredentials + .Then(credentials => new PackageSourceCredential(uriAsString, credentials.username, credentials.pat, true, string.Empty)); + + if (maybePackageSourceCredential.Succeeded) + { + logger.AppendLine($"Authentication successfully acquired for {uri.AbsoluteUri}"); + + packageSource.Credentials = maybePackageSourceCredential.Result; + } + else + { + logger.AppendLine($"Failed to authenticate {uri.AbsoluteUri}: {maybePackageSourceCredential.Failure.DescribeIncludingInnerFailures()}"); + return maybePackageSourceCredential.Failure; + } + } + } + + logger.AppendLine($"Authentication not required for {uri.AbsoluteUri}"); + + var sourceRepository = Repository.Factory.GetCoreV3(packageSource); + sourceRepositories.Add(sourceRepository); + } + + return sourceRepositories; + } + + /// + /// Whether the provided Uri requires authentication + /// + public static async Task IsAuthenticationRequiredAsync(Uri uri, CancellationToken cancellationToken, StringBuilder logger) + { + logger.AppendLine($"Checking whether authentication is required for {uri.AbsoluteUri}"); + try + { + var handler = new HttpClientHandler() + { + AllowAutoRedirect = false + }; + + using (var httpClient = new HttpClient(handler)) + { + httpClient.Timeout = TimeSpan.FromSeconds(5); + + int retries = 3; + + while (true) + { + try + { + var httpResponse = await httpClient.GetAsync(uri, cancellationToken); + + + logger.AppendLine($"Response for {uri.AbsoluteUri} is: {httpResponse.StatusCode}"); + + return (httpResponse.StatusCode == System.Net.HttpStatusCode.Unauthorized || httpResponse.StatusCode == System.Net.HttpStatusCode.Redirect) && httpResponse.Headers.WwwAuthenticate.Any(); + } + catch (TaskCanceledException) + { + logger.AppendLine($"Request for {uri.AbsoluteUri} timed out. Retries left: {retries}"); + + retries--; + + // The service seems to be unavailable. Let the pipeline deal with the unavailability, since it will likely result in a better error message + if (retries == 0) + { + return false; + } + } + } + } + } + catch (Exception ex) when (ex is HttpRequestException || ex is AggregateException) + { + logger.AppendLine($"Exception for {uri.AbsoluteUri}: {ex.Message}"); + + // The service seems to be have a problem. Let's pretend it does need auth and let the rest of the pipeline deal with the unavailability, since it will likely result in a better error message + return true; + } + } + + /// + /// Whether the URI is pointing to a VSTS URI + /// + public static bool IsVSTSPackageSecureURI(Uri downloadURI) + { + return downloadURI.Scheme == "https" && + (downloadURI.Host.EndsWith("pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith("pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Returns an HTTP-based header from a PAT, which can be injected in an HTTP header request message + /// + public static AuthenticationHeaderValue GetAuthenticationHeaderFromPAT(string pat) + { + return new AuthenticationHeaderValue("Basic", + Convert.ToBase64String( + System.Text.ASCIIEncoding.ASCII.GetBytes( + string.Format("{0}:{1}", "", pat)))); + } + + /// + /// + /// + /// + /// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable + /// + public static Task> TryGetAuthenticationCredentialsAsync( + Uri uri, + CancellationToken cancellationToken) + { + return TryDiscoverCredentialProvider() + .ThenAsync(async credentialProviderPath => await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken)); + } + + /// + /// Tries to get an authentication token using a credential provider + /// + /// + /// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers + /// using an environment variable. + /// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general + /// + public static Task> TryGetAuthenticationCredentialsAsync( + Uri uri, + string credentialProviderPath, + CancellationToken cancellationToken, + bool forceJson = false) + { + // Call the provider with the requested URI in non-interactive mode. + // Some credential providers support specifying JSON as the output format. We first try without this option (some other providers will fail if specified) + // but if the provider succeeds but we can't recognize the output as a JSON one, we try again with this option set + var processInfo = new ProcessStartInfo + { + FileName = credentialProviderPath, + Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive{(forceJson? " -F JSON" : string.Empty)}", + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }; + + return RunCredentialProvider(uri, credentialProviderPath, processInfo, forceJson, cancellationToken); + } + + private static async Task> RunCredentialProvider( + Uri uri, + string credentialProviderPath, + ProcessStartInfo processInfo, + bool jsonWasForced, + CancellationToken cancellationToken) + { + var failureCommon = $"Unable to authenticate using a credential provider for '{uri}':"; + + using (var process = Process.Start(processInfo)) + { + try + { +#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead + process?.WaitForExit(); +#pragma warning restore AsyncFixer02 + } + catch (Exception e) when (e is SystemException || e is Win32Exception) + { + return new AuthenticationFailure($"{failureCommon} Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}"); + } + + if (process == null) + { + // The process was not started + return new AuthenticationFailure($"{failureCommon} Could not start credential provider process '{credentialProviderPath}'."); + } + + // Check whether the authorization succeeded + if (process.ExitCode == 0) + { + try + { + // The response should be a well-formed JSON with a 'password' entry representing the PAT + var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput), cancellationToken); + var pat = response.GetValue("Password"); + + if (pat == null) + { + return new AuthenticationFailure($"{failureCommon} Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'."); + } + + // We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT + return (response.GetValue("Username")?.ToString(), pat.ToString()); + + } + catch (JsonReaderException) + { + // The JSON is malformed. If we were already forcing JSON, then that's an error. Otherwise, we try again but now forcing JSON + if (jsonWasForced) + { + return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' response is not a well-formed JSON."); + } + + return await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken, forceJson: true); + } + } + + return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}. Details: {await process.StandardError.ReadToEndAsync()}"); + } + } + + private static Possible TryDiscoverCredentialProvider() + { + var credentialProviderPaths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH"); + + if (credentialProviderPaths == null) + { + return new AuthenticationFailure("Unable to authenticate using a credential provider: NUGET_CREDENTIALPROVIDERS_PATH is not set."); + } + + // Here we do something slightly simpler than what NuGet does and just look for the first credential + // provider we can find + string credentialProviderPath = null; + foreach (string path in credentialProviderPaths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + { + string directory = path; + + // In some cases the entry can point directly to the credential .exe. Let's treat that uniformly and enumerate its parent directory + if (FileUtilities.TryProbePathExistence(path, followSymlink: false) is var maybeExistence && + maybeExistence.Succeeded && + maybeExistence.Result == PathExistence.ExistsAsFile) + { + directory = Directory.GetParent(path).FullName; + } + + credentialProviderPath = Directory.EnumerateFiles(directory, "CredentialProvider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault(); + if (credentialProviderPath != null) + { + break; + } + } + + if (credentialProviderPath == null) + { + return new AuthenticationFailure($"Unable to authenticate using a credential provider: Credential provider was not found under '{credentialProviderPaths}'."); + } + + return credentialProviderPath; + } + } +} diff --git a/Shared/Scripts/Init.cmd b/Shared/Scripts/Init.cmd index 2b67c8fff..78dff39c4 100644 --- a/Shared/Scripts/Init.cmd +++ b/Shared/Scripts/Init.cmd @@ -114,6 +114,7 @@ REM ********************************* REM We'll conditionally set the credential provider if not set on the machine. REM If not set we will set it to the local one in the enlistment but iwth the b-drive substitution if NOT DEFINED NUGET_CREDENTIALPROVIDERS_PATH ( + ECHO NUGET_CREDENTIALPROVIDERS_PATH not set. Setting it to %TOOLROOT% set NUGET_CREDENTIALPROVIDERS_PATH=%TOOLROOT% ) goto :EOF diff --git a/Shared/Tools/Microsoft.IdentityModel.Clients.ActiveDirectory.dll b/Shared/Tools/Microsoft.IdentityModel.Clients.ActiveDirectory.dll new file mode 100644 index 000000000..2a22068ce Binary files /dev/null and b/Shared/Tools/Microsoft.IdentityModel.Clients.ActiveDirectory.dll differ diff --git a/Shared/Tools/Newtonsoft.Json.dll b/Shared/Tools/Newtonsoft.Json.dll new file mode 100644 index 000000000..2d82d012d Binary files /dev/null and b/Shared/Tools/Newtonsoft.Json.dll differ diff --git a/Shared/Tools/NuGet.Common.dll b/Shared/Tools/NuGet.Common.dll new file mode 100644 index 000000000..81752eb1f Binary files /dev/null and b/Shared/Tools/NuGet.Common.dll differ diff --git a/Shared/Tools/NuGet.Protocol.dll b/Shared/Tools/NuGet.Protocol.dll new file mode 100644 index 000000000..8b715dcaa Binary files /dev/null and b/Shared/Tools/NuGet.Protocol.dll differ diff --git a/Shared/Tools/PowerArgs.dll b/Shared/Tools/PowerArgs.dll new file mode 100644 index 000000000..1621345b4 Binary files /dev/null and b/Shared/Tools/PowerArgs.dll differ diff --git a/Shared/Tools/readme.txt b/Shared/Tools/readme.txt new file mode 100644 index 000000000..042baeaea --- /dev/null +++ b/Shared/Tools/readme.txt @@ -0,0 +1,3 @@ +These files are used by: +- bxl.ps1 in order to retrieve the latest bxl.exe from the corresponding feed and bootstrap the build with the latest LKG. Nuget.exe and the checked-in credential provider are used to authenticate against the VSTS feed. +- bxl.ps1 script also sets NUGET_CREDENTIALPROVIDERS_PATH environment variable to point to this folder (if not previously set). That implies that the Nuget resolver, if configured, will also use this credential provider to authenticate. \ No newline at end of file diff --git a/cg/nuget/cgmanifest.json b/cg/nuget/cgmanifest.json index 82d3ef71e..58cedb3d3 100644 --- a/cg/nuget/cgmanifest.json +++ b/cg/nuget/cgmanifest.json @@ -2742,7 +2742,25 @@ "Type": "NuGet", "NuGet": { "Name": "NuGet.CommandLine", - "Version": "4.7.1" + "Version": "5.11.0" + } + } + }, + { + "Component": { + "Type": "NuGet", + "NuGet": { + "Name": "NuGet.Common", + "Version": "5.11.0" + } + } + }, + { + "Component": { + "Type": "NuGet", + "NuGet": { + "Name": "NuGet.Configuration", + "Version": "5.11.0" } } }, @@ -2751,7 +2769,25 @@ "Type": "NuGet", "NuGet": { "Name": "NuGet.Frameworks", - "Version": "5.0.0" + "Version": "5.11.0" + } + } + }, + { + "Component": { + "Type": "NuGet", + "NuGet": { + "Name": "NuGet.Packaging", + "Version": "5.11.0" + } + } + }, + { + "Component": { + "Type": "NuGet", + "NuGet": { + "Name": "NuGet.Protocol", + "Version": "5.11.0" } } }, @@ -2760,7 +2796,7 @@ "Type": "NuGet", "NuGet": { "Name": "NuGet.Versioning", - "Version": "4.6.0" + "Version": "5.11.0" } } }, diff --git a/config.dsc b/config.dsc index 9a9da885e..d165613d7 100644 --- a/config.dsc +++ b/config.dsc @@ -38,11 +38,10 @@ config({ ] }, { + // The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH. kind: "Nuget", - // This configuration pins people to a specific version of nuget - // The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH. TODO: It can be alternatively pinned here, - // but when it fails to download (e.g. from a share) the build is aborted. Consider making the failure non-blocking. + // TODO: remove once the new LKG gets deployed, nuget.exe is not used anymore configuration: { toolUrl: "https://dist.nuget.org/win-x86-commandline/v4.9.4/NuGet.exe", hash: "VSO0:17E8C8C0CDCCA3A6D1EE49836847148C4623ACEA5E6E36E10B691DA7FDC4C39200" @@ -60,10 +59,7 @@ config({ } : { "buildxl-selfhost" : "https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL.Selfhost/nuget/v3/index.json", - "nuget.org" : "http://api.nuget.org/v3/index.json", - "roslyn-tools" : "https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json", - "msbuild" : "https://dotnet.myget.org/F/msbuild/api/v3/index.json", - "dotnet-core" : "https://dotnet.myget.org/F/dotnet-core/api/v3/index.json", + "nuget.org" : "https://api.nuget.org/v3/index.json", "dotnet-arcade" : "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json", }, @@ -161,9 +157,13 @@ config({ { id: "System.Threading.Tasks.Dataflow", version: "4.9.0" }, // Nuget - { id: "NuGet.CommandLine", version: "4.7.1" }, - { id: "NuGet.Versioning", version: "4.6.0" }, // Can't use the latest becuase nuget extracts to folder with metadata which we don't support yet. - { id: "NuGet.Frameworks", version: "5.0.0"}, // needed for qtest on .net core + { id: "NuGet.Packaging", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData", "System.Security.Cryptography.Pkcs"] }, + { id: "NuGet.Configuration", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData"] }, + { id: "NuGet.Common", version: "5.11.0" }, + { id: "NuGet.Protocol", version: "5.11.0" }, + { id: "NuGet.Versioning", version: "5.11.0" }, + { id: "NuGet.CommandLine", version: "5.11.0" }, + { id: "NuGet.Frameworks", version: "5.11.0"}, // needed for qtest on .net core // ProjFS (virtual file system) { id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },