Merged PR 662608: Reapply NuGet resolver schedules real pips

Same commit as before plus:
* Force check the nuget package fp on disk (apparently this was off by default because dotnet core didn't support nuget at that time) (that should fix the DFA we saw, which is a layout difference between a public vs internal package when an agent gets reused)
* Capture stderr when executing the cred provider, for better error diagnostics.

From the issues we saw, there is still one that I was not able to repro (https://dev.azure.com/mseng/Domino/_build/results?buildId=17296398&view=logs&j=cfa20e98-6997-523c-4233-f0a7302c929f&t=144c797d-20c9-5763-9672-cca1a82cd366). I'll keep trying, but the stderr should be able to tell us more if that happens again. Hopefully we can spot this before I merge this.

Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc

@@ -179,9 +179,9 @@ export function evaluate(args: Arguments): Result {
         ],
         allowedSurvivingChildProcessNames: [
             "mspdbsrv.exe",
-            "VCTIP.exe",
             "conhost.exe",
-        ]
+        ],
+        unsafe: { childProcessesToBreakawayFromSandbox: [a`vctip.exe`] }
     });
 
     let midlOutput = <Midl.Result>{

Public/Sdk/Public/Managed/Tools/NuGet/empty.config

@@ -27,13 +27,16 @@ Each time a new nuget version is updated we must make sure that any configuratio
     <solution>
         <clear />
     </solution>
-    <packageSourceCredentials>
-        <clear />
-    </packageSourceCredentials>
     <apikeys>
         <clear />
     </apikeys>
     <activePackageSource>
         <clear />
     </activePackageSource>
+    <fallbackPackageFolders>
+        <clear />
+    </fallbackPackageFolders>
+    <packageSourceMapping>
+        <clear />
+    </packageSourceMapping>
 </configuration>

Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc

@@ -14,7 +14,8 @@ export const tool : Transformer.ToolDefinition = {
     exe: Nuget.Contents.all.getFile(r`tools/NuGet.exe`),
     description: "NuGet pack",
     untrackedDirectoryScopes: [
-        d`${Context.getMount("ProgramData").path}/Nuget`
+        d`${Context.getMount("ProgramData").path}/Nuget`,
+        ...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("ProgramFilesX86").path}/Nuget`]),
     ],
     dependsOnWindowsDirectories: true,
     dependsOnAppDataDirectory: true,

Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc

@@ -839,7 +839,11 @@ interface UntrackingSettings {
 }
 
 interface NuGetConfiguration extends ToolConfiguration {
-    credentialProviders?: ToolConfiguration[];
+    /**
+     * The download timeout, in minutes, for each NuGet download pip. Defaults to 20m.
+     * Equivalent to configuring timeoutInMilliseconds for the corresponding Transformers.ToolDefinition
+     */
+    downloadTimeoutMin?: number
 }
 
 interface ScriptResolverDefaults {

Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc

@@ -181,9 +181,9 @@ function runTest(args : TestRunArguments) : File[] {
         testAssembly: args.testDeployment.primaryFile.path,
         qTestType: Qtest.QTestType.msTest_latest,
         qTestDirToDeploy: args.testDeployment.contents,
-        qTestAdapterPath: Transformer.sealDirectory({
+        qTestAdapterPath: Transformer.sealPartialDirectory({
             root: testAdapterPath, 
-            files: globR(testAdapterPath, "*")
+            files: rootTestAdapterPath.contents.filter(f => f.path.isWithin(testAdapterPath)),
         }),
         qTestDotNetFramework: getQTestDotNetFramework(),
         qTestPlatform: Qtest.QTestPlatform.x64,

Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs

@@ -23,10 +23,24 @@ namespace BuildXL.Engine
         {
         }
 
-        /// <nodoc />
-        public void SetMountsTable(MountsTable mountsTable)
+        /// <summary>
+        /// Adds all the mounts specified in the given mounts table
+        /// </summary>
+        public void UpdateMountsTable(MountsTable mountsTable)
         {
-            m_customMountsTable = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount);
+            var allMounts = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount);
+
+            if (m_customMountsTable == null)
+            {
+                m_customMountsTable = allMounts;
+            }
+            else
+            {
+                foreach (var mount in allMounts)
+                {
+                    m_customMountsTable[mount.Key] = mount.Value;
+                }
+            }
         }
 
         /// <summary>
@@ -41,7 +55,7 @@ namespace BuildXL.Engine
                 return false;
             }
 
-            SetMountsTable(mountsTable);
+            UpdateMountsTable(mountsTable);
 
             return true;
         }

Public/Src/Engine/Dll/FrontEndEngineImplementation.cs

@@ -93,6 +93,7 @@ namespace BuildXL.Engine
         private readonly MountsTable m_mountsTable;
 
         private readonly LoggingContext m_loggingContext;
+        private readonly DirectoryTranslator m_directoryTranslator;
 
         /// <summary>
         /// Creates an instance of <see cref="FrontEndEngineImplementation"/>.
@@ -130,6 +131,7 @@ namespace BuildXL.Engine
             m_snapshotCollector = snapshotCollector;
             GetTimerUpdatePeriod = timerUpdatePeriod;
             Layout = configuration.Layout;
+            m_directoryTranslator = directoryTranslator;
 
             if (ShouldUseSpecCache(configuration))
             {
@@ -184,6 +186,12 @@ namespace BuildXL.Engine
                 });
         }
 
+        /// <inheritdoc/>
+        public override AbsolutePath Translate(AbsolutePath path)
+        {
+            return m_directoryTranslator?.Translate(path, PathTable) ?? path;
+        }
+
         private bool ShouldUseSpecCache(IConfiguration configuration)
         {
             var sourceDirectory = configuration.Layout.SourceDirectory.ToString(PathTable);

Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs

@@ -197,7 +197,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult
         }
 
 
-        protected void AddSdk(string sdkLocation)
+        protected void AddSdk(string relativeSdkLocation)
         {
             Configuration.Resolvers.Add(
                 new SourceResolverSettings
@@ -206,7 +206,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult
                     Modules = new List<DiscriminatingUnion<AbsolutePath, IInlineModuleDefinition>>
                               {
                                   new DiscriminatingUnion<AbsolutePath, IInlineModuleDefinition>(
-                                      AbsolutePath.Create(Context.PathTable, sdkLocation)),
+                                      AbsolutePath.Create(Context.PathTable, TestDeploymentDir).Combine(Context.PathTable, RelativePath.Create(Context.StringTable, relativeSdkLocation))),
                               },
                 });
         }

Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs

@@ -35,7 +35,6 @@ namespace Test.BuildXL.Engine
     {
         private CacheInitializer m_cacheInitializer;
 
-
         /* The build used for tests in this class generate the following layout
          *
          * \HelloWorldFinal.exe.config (source)

Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc

@@ -3,6 +3,8 @@
 
 import * as DetoursServices from "BuildXL.Sandbox.Windows";
 import * as Xml from "Sdk.Xml";
+import * as Deployment from "Sdk.Deployment";
+import {Transformer} from "Sdk.Transformers";
 
 namespace Engine {
 
@@ -23,8 +25,34 @@ namespace Engine {
         },
     ];
 
-    // Update the value of this variable if you change the version of Microsoft.Net.Compilers in config.dsc.
-    const microsoftNetCompilerSpec = f`${Context.getMount("FrontEnd").path}/Nuget/specs/Microsoft.Net.Compilers/4.0.1/module.config.bm`;
+    // We generate specs as if the compilers package was source files, but point it to the downloaded NuGet
+    const compilerSpecsDir = Context.getNewOutputDirectory("compilers-specs");
+    const microsoftNetCompilerSpec = Transformer.writeAllText(p`${compilerSpecsDir}/module.config.bm`, 
+        "module({name: 'Microsoft.Net.Compilers', version: '4.0.1', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences});");
+    const specFile = Transformer.writeAllLines(p`${compilerSpecsDir}/package.dsc`, [
+        "import {Transformer} from 'Sdk.Transformers';",
+        "namespace Contents {",
+            "export declare const qualifier: {};",
+            "@@public export const all: StaticDirectory = Transformer.sealPartialDirectory(d`package`, globR(d`package`, '*'));",
+        "}"
+    ]);
+
+    // Deploy the compilers package plus the specs that refer to it
+    const deployable : Deployment.Definition = {
+        contents: [
+            {
+                subfolder: a`compilers`,
+                contents: [
+                    {
+                         subfolder: a`package`, 
+                         contents: [importFrom('Microsoft.Net.Compilers').Contents.all]
+                    }, 
+                    specFile, 
+                    microsoftNetCompilerSpec
+                ]
+            }
+        ]
+    };
 
     @@public
     export const categoriesToRunInParallel = [
@@ -47,12 +75,11 @@ namespace Engine {
             },
             parallelGroups: categoriesToRunInParallel,
             testRunData: {
-                MicrosoftNetCompilersSdkLocation: microsoftNetCompilerSpec,
+                MicrosoftNetCompilersSdkLocation: "compilers/module.config.bm",
             },
             tools: {
                 exec: {
                     dependencies: [
-                        microsoftNetCompilerSpec,
                         importFrom("Microsoft.Net.Compilers").Contents.all,
                         importFrom("Microsoft.NETCore.Compilers").Contents.all,
                     ]
@@ -87,6 +114,7 @@ namespace Engine {
         ],
         runtimeContent: [
             ...libsUsedForTesting,
+            deployable
         ],
     });
 }

Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs

@@ -882,6 +882,7 @@ namespace Test.BuildXL.Scheduler
             bool preservePathSetCasing = source.Vary(p => p.PreservePathSetCasing);
             bool writingToStandardErrorFailsExecution = source.Vary(p => p.WritingToStandardErrorFailsExecution);
             bool disableFullReparsePointResolving = source.Vary(p => p.DisableFullReparsePointResolving);
+            bool bypassFingerprintSalt = source.Vary(p => p.BypassFingerprintSalt);
 
             Process.Options options = Process.Options.None;
             if (hasUntrackedChildProcesses)
@@ -940,6 +941,11 @@ namespace Test.BuildXL.Scheduler
                 options |= Process.Options.DisableFullReparsePointResolving;
             }
 
+            if (bypassFingerprintSalt)
+            {
+                options |= Process.Options.BypassFingerprintSalt;
+            }
+
             return new Process(
                 executable: executable,
                 workingDirectory: workingDirectory,

Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs

@@ -52,6 +52,7 @@ namespace BuildXL.FrontEnd.Core
             string weakPackageFingerprint,
             PackageIdentity package,
             AbsolutePath packageTargetFolder,
+            AbsolutePath pathToNuspec,
             Func<Task<Possible<IReadOnlyList<RelativePath>>>> producePackage)
         {
             // Check if we can reuse a package that is layed out on disk already.
@@ -97,6 +98,7 @@ namespace BuildXL.FrontEnd.Core
                         weakPackageFingerprint,
                         package,
                         packageTargetFolder,
+                        pathToNuspec,
                         possiblePackageContents.Result),
                     justification: "Okay to ignore putting in cache failure, will happen next time"
                 );
@@ -208,7 +210,10 @@ namespace BuildXL.FrontEnd.Core
                 return false;
             }
 
-            Logger.PackagePresumedUpToDateWithoutHashComparison(LoggingContext, package.GetFriendlyName());
+            if (!FrontEndConfiguration.RespectWeakFingerprintForNugetUpToDateCheck())
+            {
+                Logger.PackagePresumedUpToDateWithoutHashComparison(LoggingContext, package.GetFriendlyName());
+            }
 
             return true;
         }
@@ -354,7 +359,7 @@ namespace BuildXL.FrontEnd.Core
 
             // Saving package's hash file on disk.      
             // But we can do this only when the content is not empty.
-                var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList();
+            var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList();
 
             if (packagesContent.Count == 0)
             {
@@ -363,9 +368,6 @@ namespace BuildXL.FrontEnd.Core
                 return PackageDownloadResult.RecoverableError(package);
             }
 
-            var newPackageHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, packagesContent);
-            TryUpdatePackageHashFile(loggingContext, package, packageHashFile, packageHash, newPackageHash);
-
             m_logger.PackageRestoredFromCache(loggingContext, package.GetFriendlyName());
 
             if (forcePopulatePackageCache)
@@ -394,11 +396,20 @@ namespace BuildXL.FrontEnd.Core
                 }
             }
 
+            // The hash file is stored as part of the package, so we have the package layout available there
+            var maybePackageHashFile = PackageHashFile.TryReadFrom(GetPackageHashFile(packageTargetFolder));
+
+            if (!maybePackageHashFile.Succeeded)
+            {
+                m_logger.DownloadPackageFailedDueToCacheError(loggingContext, friendlyName, maybePackageHashFile.Failure.Describe());
+                return PackageDownloadResult.RecoverableError(package);
+            }
+
             // Step: Return descriptor indicating it was successfully restored from the cache
             return PackageDownloadResult.FromCache(
                 package,
                 packageTargetFolder,
-                packageDescriptor.Contents.Select(c => RelativePath.Create(FrontEndContext.StringTable, c.Key)).ToList(),
+                maybePackageHashFile.Result.Content.Select(entry => RelativePath.Create(FrontEndContext.StringTable, entry)).ToList(),
                 weakPackageFingerprintHash.Hash.ToHex());
         }
 
@@ -407,6 +418,7 @@ namespace BuildXL.FrontEnd.Core
             string weakPackageFingerprint,
             PackageIdentity package,
             AbsolutePath packageTargetFolder,
+            AbsolutePath pathToNuspec,
             IReadOnlyList<RelativePath> packageContent)
         {
             var friendlyName = package.GetFriendlyName();
@@ -423,11 +435,22 @@ namespace BuildXL.FrontEnd.Core
             // Cache was already initialized
             var cache = await m_nugetCache;
 
-            // Step: Store all the files into the content cache
+            // Generate the hash file, since that is stored as part of the cache content
+            var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint);
+            // The content should have relative paths
+            var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList();
+            var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content);
+            var packageHashFile = GetPackageHashFile(packageTargetFolder);
+            TryUpdatePackageHashFile(loggingContext, package, packageHashFile, oldHash: null, newHash: newHash);
+
+            // Step: Store the nuspec and the hash file into the content cache
             var stringKeyedHashes = new List<StringKeyedHash>();
-            foreach (var relativePath in packageContents)
+            
+            foreach (var absolutePath in new[] { pathToNuspec, AbsolutePath.Create(PathTable, packageHashFile)})
             {
-                var targetFileLocation = packageTargetFolder.Combine(PathTable, relativePath).Expand(PathTable);
+                var targetFileLocation = absolutePath.Expand(PathTable);
+                var result = packageTargetFolder.TryGetRelative(PathTable, absolutePath, out var relativePath);
+                Contract.Assert(result);
 
                 ContentHash contentHash;
                 try
@@ -453,7 +476,6 @@ namespace BuildXL.FrontEnd.Core
                 }
             }
 
-            var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint);
             // Step: Create a descriptor and store that in the fingerprint store under the weak fingerprint.
             var cacheDescriptor = PackageDownloadDescriptor.Create(
                 friendlyName,
@@ -471,11 +493,6 @@ namespace BuildXL.FrontEnd.Core
 
             m_logger.PackageNotFoundInCacheAndDownloaded(loggingContext, package.Id, package.Version, weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint);
 
-            // The content should have relative paths
-            var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList();
-            var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content);
-            TryUpdatePackageHashFile(loggingContext, package, GetPackageHashFile(packageTargetFolder), oldHash: null, newHash: newHash);
-
             return Unit.Void;
         }
 

Public/Src/FrontEnd/Core/NugetCache.cs

@@ -132,7 +132,7 @@ namespace BuildXL.FrontEnd.Core
         private static ContentFingerprint CreateDownloadFingerprint(string baseText)
         {
             // In case something in the cached Bond data becomes incompatible, we must not match.
-            const string VersionText = ", BondDataVersion=2;FingerprintVersion=1";
+            const string VersionText = ", BondDataVersion=2;FingerprintVersion=5";
             var fingerprint = FingerprintUtilities.Hash(baseText + VersionText);
             return new ContentFingerprint(fingerprint);
         }

Public/Src/FrontEnd/Core/PackageHashFile.cs

@@ -31,7 +31,7 @@ namespace BuildXL.FrontEnd.Core
 
         // The file format change will force specs regeneration.
         // Change the version if the nuget spec generation has changed in a backward incompatible way.
-        private const string HashFileFormatVersion = "9";
+        private const string HashFileFormatVersion = "10";
 
         /// <summary>
         /// The minimal number of lines for the hash file.

Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc

@@ -14,9 +14,11 @@ namespace Nuget {
         sources: globR(d`.`, "*.cs"),
         references: [
             ...addIf(BuildXLSdk.isFullFramework,
+                NetFx.Netstandard.dll,
+                NetFx.System.IO.Compression.dll,
                 NetFx.System.Xml.dll,
                 NetFx.System.Xml.Linq.dll,
-                NetFx.Netstandard.dll
+                NetFx.System.Net.Http.dll
             ),
             ...addIf(BuildXLSdk.isFullFramework,
                 importFrom("System.Memory").withQualifier({targetFramework: "netstandard2.0"}).pkg
@@ -33,6 +35,7 @@ namespace Nuget {
             importFrom("BuildXL.Engine").Processes.dll,
             importFrom("BuildXL.Pips").dll,
             importFrom("BuildXL.Utilities").dll,
+            importFrom("BuildXL.Utilities").VstsAuthentication.dll,
             importFrom("BuildXL.Utilities").Collections.dll,
             importFrom("BuildXL.Utilities").Configuration.dll,
             importFrom("BuildXL.Utilities").Interop.dll,
@@ -42,12 +45,21 @@ namespace Nuget {
             importFrom("BuildXL.Utilities").Script.Constants.dll,
 
             importFrom("Newtonsoft.Json").pkg,
-            importFrom("NuGet.Versioning").pkg,
+            importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
 
             ...BuildXLSdk.tplPackages,
         ],
+        runtimeContent: [
+            // Keep in sync with path at Public\Sdk\Public\Tools\NugetDownloader\Tool.NugetDownloader.dsc
+            importFrom("BuildXL.Tools").NugetDownloader.dll
+        ],
         internalsVisibleTo: [
             "Test.BuildXL.FrontEnd.Nuget"
-        ]
+        ],
     });
 }

Public/Src/FrontEnd/Nuget/Failures/NugetFailedWithNonZeroExitCodeFailure.cs

@@ -1,38 +0,0 @@
-// Copyright (c) Microsoft Corporation.
-// Licensed under the MIT License.
-
-using System;
-using BuildXL.Utilities.Configuration;
-using static BuildXL.Utilities.FormattableStringEx;
-namespace BuildXL.FrontEnd.Nuget
-{
-    /// <summary>
-    /// Nuget.exe failed with non-zero exit code.
-    /// </summary>
-    public sealed class NugetFailedWithNonZeroExitCodeFailure : NugetFailure
-    {
-        private readonly INugetPackage m_package;
-        private readonly int m_exitCode;
-        private readonly string m_stdOut;
-        private readonly string m_stdErr;
-
-        /// <nodoc />
-        public NugetFailedWithNonZeroExitCodeFailure(INugetPackage package, int exitCode, string stdOut, string stdErr)
-            : base(FailureType.PackageNotFound)
-        {
-            m_package = package;
-            m_exitCode = exitCode;
-
-            m_stdOut = stdOut?.Trim();
-            m_stdErr = stdErr?.Trim();
-        }
-
-        /// <inheritdoc />
-        public override string Describe()
-        {
-            var separator = !string.IsNullOrEmpty(m_stdOut) && !string.IsNullOrEmpty(m_stdErr) ? Environment.NewLine : string.Empty;
-            var output = I($"{m_stdOut}{separator}{m_stdErr}");
-            return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored because nuget.exe failed with exit code '{m_exitCode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details.");
-        }
-    }
-}

Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics.ContractsLight;
+using System.Linq;
 using BuildXL.Utilities;
 using BuildXL.Utilities.Configuration;
 using TypeScript.Net.Extensions;
@@ -21,13 +22,21 @@ namespace BuildXL.FrontEnd.Nuget
         public readonly FailureType Type;
 
         /// <nodoc />
-        public readonly Exception Exception;
+        public readonly string Message;
 
         /// <nodoc />
         public NugetFailure(FailureType failureType, Exception e = null)
         {
             Type = failureType;
-            Exception = e;
+
+            Message = GetAllMessages(e);
+        }
+
+        /// <nodoc />
+        public NugetFailure(FailureType failureType, string message)
+        {
+            Type = failureType;
+            Message = message;
         }
 
         /// <nodoc />
@@ -58,20 +67,28 @@ namespace BuildXL.FrontEnd.Nuget
         }
 
         /// <summary>
-        /// Nuget.exe failed to restore a package.
+        /// Nuget invocation failure
         /// </summary>
-        public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, int exitCode, string stdOut, string stdErr)
+        public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, Exception e)
         {
-            Contract.Requires(package != null);
-            Contract.Requires(exitCode != 0);
+            return CreateNugetInvocationFailure(package, GetAllMessages(e));
+        }
+
+        /// <summary>
+        /// Nuget invocation failure
+        /// </summary>
+        public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, string message)
+        {
+            Contract.RequiresNotNull(package);
+            Contract.RequiresNotNull(message);
 
             // If the stdOut has the following text: 'NotFound http' or 'WARNING: Unable to find version', it means that the package name or version are not found.
-            if (stdOut.Contains("NotFound http") || stdOut.Contains("WARNING: Unable to find version"))
+            if (message.Contains("NotFound http") || message.Contains("WARNING: Unable to find version"))
             {
                 return new CanNotFindPackageFailure(package);
             }
 
-            return new NugetFailedWithNonZeroExitCodeFailure(package, exitCode, stdOut, stdErr);
+            return new NugetInvocationFailure(package, message);
         }
 
         /// <summary>
@@ -89,10 +106,10 @@ namespace BuildXL.FrontEnd.Nuget
         {
             if (Package != null)
             {
-                return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Exception?.ToStringDemystified()}");
+                return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Message}");
             }
 
-            return I($"Failed to process nuget packages due to {Type.ToString()}. {Exception?.ToStringDemystified()}");
+            return I($"Failed to process nuget packages due to {Type.ToString()}. {Message}");
         }
 
         /// <inheritdoc />
@@ -110,30 +127,15 @@ namespace BuildXL.FrontEnd.Nuget
         /// <nodoc />
         public enum FailureType
         {
-            /// <nodoc />
-            FetchNugetExe,
-
             /// <nodoc />
             FetchCredentialProvider,
 
-            /// <nodoc />
-            WriteConfigFile,
-
             /// <nodoc />
             WriteSpecFile,
 
             /// <nodoc />
             CleanTargetFolder,
 
-            /// <nodoc />
-            NugetFailedWithNonZeroExitCode,
-
-            /// <nodoc />
-            NugetFailedWithIoException,
-
-            /// <nodoc />
-            ListPackageContents,
-
             /// <nodoc />
             ReadNuSpecFile,
 
@@ -160,6 +162,37 @@ namespace BuildXL.FrontEnd.Nuget
 
             /// <nodoc />
             UnhandledError,
+
+            /// <nodoc/>
+            NoBaseAddressForRepository,
+        }
+
+        private static string GetAllMessages(Exception e)
+        {
+            if (e is null)
+            {
+                return string.Empty;
+            }
+
+            if (e is AggregateException aggregateException)
+            {
+                return string.Join(Environment.NewLine, aggregateException.Flatten().InnerExceptions.SelectMany(ie => GetInnerExceptions(ie)).Select(ex => ex.Message));
+            }
+            else
+            {
+                return string.Join(Environment.NewLine, GetInnerExceptions(e).Select(e => e.Message));
+            }
+        }
+
+        private static IEnumerable<Exception> GetInnerExceptions(Exception ex)
+        {
+            var innerException = ex;
+            do
+            {
+                yield return innerException;
+                innerException = innerException.InnerException;
+            }
+            while (innerException != null);
         }
     }
 }

Public/Src/FrontEnd/Nuget/Failures/NugetInvocationFailure.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using BuildXL.Utilities.Configuration;
+using static BuildXL.Utilities.FormattableStringEx;
+namespace BuildXL.FrontEnd.Nuget
+{
+    /// <summary>
+    /// Nuget invocation failed.
+    /// </summary>
+    public sealed class NugetInvocationFailure : NugetFailure
+    {
+        private readonly INugetPackage m_package;
+        private readonly string m_message;
+
+        /// <nodoc />
+        public NugetInvocationFailure(INugetPackage package, string message)
+            : base(FailureType.PackageNotFound)
+        {
+            m_package = package;
+            
+            m_message = message?.Trim();
+        }
+
+        /// <inheritdoc />
+        public override string Describe()
+        {
+            return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored. {m_message}.");
+        }
+    }
+}

Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs

@@ -38,6 +38,14 @@ namespace BuildXL.FrontEnd.Nuget
 
         /// <nodoc />
         public MultiValueDictionary<Moniker, INugetPackage> DependenciesPerFramework { get; }
+        
+        /// <summary>
+        /// Credential provider path that is used to retrieve the package
+        /// </summary>
+        /// <remarks>
+        /// AbsolutePath.Invalid if none is to be used
+        /// </remarks>
+        public AbsolutePath CredentialProviderPath { get; }
 
         /// <nodoc />
         public bool IsManagedPackage { get; set; }
@@ -129,7 +137,8 @@ namespace BuildXL.FrontEnd.Nuget
             NugetFrameworkMonikers nugetFrameworkMonikers,
             PackageOnDisk packageOnDisk,
             Dictionary<string, INugetPackage> packagesOnConfig,
-            bool doNotEnforceDependencyVersions)
+            bool doNotEnforceDependencyVersions,
+            AbsolutePath credentialProviderPath)
         {
             m_context = context;
             PackageOnDisk = packageOnDisk;
@@ -142,6 +151,7 @@ namespace BuildXL.FrontEnd.Nuget
             AssemblyToTargetFramework = new MultiValueDictionary<PathAtom, NugetTargetFramework>();
             m_dependencies = new List<INugetPackage>();
             DependenciesPerFramework = new MultiValueDictionary<PathAtom, INugetPackage>();
+            CredentialProviderPath = credentialProviderPath;
         }
 
         /// <summary>
@@ -156,13 +166,14 @@ namespace BuildXL.FrontEnd.Nuget
             [CanBeNull] XDocument nuSpec,
             PackageOnDisk packageOnDisk,
             Dictionary<string, INugetPackage> packagesOnConfig,
-            bool doNotEnforceDependencyVersions)
+            bool doNotEnforceDependencyVersions,
+            AbsolutePath credentialProviderPath)
         {
             Contract.Requires(context != null);
             Contract.Requires(packageOnDisk != null);
 
             var analyzedPackage = new NugetAnalyzedPackage(context, nugetFrameworkMonikers, packageOnDisk,
-                packagesOnConfig, doNotEnforceDependencyVersions);
+                packagesOnConfig, doNotEnforceDependencyVersions, credentialProviderPath);
 
             analyzedPackage.ParseManagedSemantics();
             if (nuSpec != null && !analyzedPackage.TryParseDependenciesFromNuSpec(nuSpec))

Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs

@@ -41,7 +41,7 @@ namespace BuildXL.FrontEnd.Nuget
         public IReadOnlyCollection<string> SupportedResolvers { get; } = new[] { WorkspaceNugetModuleResolver.NugetResolverName };
 
         /// <inheritdoc />
-        public bool ShouldRestrictBuildParameters { get; } = true;
+        public bool ShouldRestrictBuildParameters { get; } = false;
 
         /// <inheritdoc />
         public void InitializeFrontEnd(FrontEndHost host, FrontEndContext context, IConfiguration configuration)

Public/Src/FrontEnd/Nuget/NugetInspectedPackage.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
+using BuildXL.Utilities;
+
+namespace BuildXL.FrontEnd.Nuget
+{
+    /// <summary>
+    /// The result of inspecting a package with <see cref="NugetPackageInspector"/>
+    /// </summary>
+    public readonly struct NugetInspectedPackage
+    {
+        /// <nodoc/>
+        public readonly string Nuspec { get; }
+
+        /// <nodoc/>
+        public readonly IReadOnlyList<RelativePath> Content { get; }
+
+        /// <nodoc/>
+        public NugetInspectedPackage(string nuspec, IReadOnlyList<RelativePath> content)
+        {
+            Contract.RequiresNotNull(nuspec);
+            Contract.RequiresNotNull(content);
+
+            Nuspec = nuspec;
+            Content = content;
+        }
+    }
+}

Public/Src/FrontEnd/Nuget/NugetPackageInspector.cs

@@ -0,0 +1,365 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using BuildXL.FrontEnd.Nuget.Tracing;
+using BuildXL.Utilities;
+using BuildXL.Utilities.Configuration;
+using BuildXL.Utilities.Instrumentation.Common;
+using BuildXL.Utilities.VstsAuthentication;
+using NuGet.Configuration;
+using NuGet.Packaging;
+using NuGet.Protocol;
+
+namespace BuildXL.FrontEnd.Nuget
+{
+    /// <summary>
+    /// Retrieves NuGet package layouts and specs from a set of feeds
+    /// </summary>
+    /// <remarks>
+    /// Packages are not necessarily downloaded in full in order to inspect them, and partial downloads are always attempted first
+    /// </remarks>
+    public class NugetPackageInspector
+    {
+        // A 5K chunk looks reasonable as the initial download: most package central directory fits in 5K, so no extra requests are needed
+        private const long MinimalChunkSizeInBytes = 5_000;
+        // On every partial download iteration we linearly grow using a 20K base (i.e. 20, 40, 60, 80, etc.)
+        private const long IncrementChunkSizeInBytes = 20_000;
+
+        private readonly CancellationToken m_cancellationToken;
+        private readonly LoggingContext m_loggingContext;
+        private readonly IEnumerable<(string repositoryName, Uri repositoryUri)> m_repositories;
+        private readonly StringTable m_stringTable;
+        private readonly Func<Possible<string>> m_discoverCredentialProvider;
+        private IReadOnlyDictionary<Uri, PackageSourceCredential> m_packageBaseAddress;
+        private readonly Lazy<Task<Possible<bool>>> m_initializationResult;
+
+        /// <nodoc/>
+        public NugetPackageInspector(
+            IEnumerable<(string repositoryName, Uri repositoryUri)> repositories, 
+            StringTable stringTable, 
+            Func<Possible<string>> discoverCredentialProvider, 
+            CancellationToken cancellationToken, 
+            Utilities.Instrumentation.Common.LoggingContext loggingContext)
+        {
+            Contract.RequiresNotNull(repositories);
+            Contract.RequiresNotNull(discoverCredentialProvider);
+
+            m_cancellationToken = cancellationToken;
+            m_loggingContext = loggingContext;
+            m_repositories = repositories;
+            m_stringTable = stringTable;
+            m_discoverCredentialProvider = discoverCredentialProvider;
+
+            m_initializationResult = new Lazy<Task<Possible<bool>>>(async () => {
+                
+                var logger = new StringBuilder();
+                
+                try
+                {
+                    return (await VSTSAuthenticationHelper.TryCreateSourceRepositories(m_repositories, m_discoverCredentialProvider, m_cancellationToken, logger))
+                        .Then<bool>(sourceRepositories =>
+                        {
+                            var packageBaseAddressMutable = new Dictionary<Uri, PackageSourceCredential>();
+                            foreach (var sourceRepository in sourceRepositories)
+                            {
+                                var serviceIndexResource = sourceRepository.GetResource<ServiceIndexResourceV3>(m_cancellationToken);
+
+                                if (serviceIndexResource == null)
+                                {
+                                    return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, $"Cannot find index service for ${sourceRepository.PackageSource.SourceUri}");
+                                }
+
+                                foreach (Uri packageBaseAddress in serviceIndexResource.GetServiceEntryUris(ServiceTypes.PackageBaseAddress))
+                                {
+                                    packageBaseAddressMutable[packageBaseAddress] = sourceRepository.PackageSource.Credentials;
+                                }
+                            }
+
+                            m_packageBaseAddress = packageBaseAddressMutable;
+
+                            return true;
+                        });
+
+                }
+                catch (Exception e) when (e is AggregateException || e is HttpRequestException)
+                {
+                    return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, e);
+                }
+                finally
+                {
+                    var log = logger.ToString();
+                    if (!string.IsNullOrWhiteSpace(log))
+                    {
+                        Logger.Log.NuGetInspectionInitializationInfo(m_loggingContext, log);
+                    }
+                }
+            },
+            LazyThreadSafetyMode.ExecutionAndPublication);
+        }
+
+        /// <summary>
+        /// Whether <see cref="TryInitAsync"/> has been succesfully called
+        /// </summary>
+        /// <remarks>Thread safe</remarks>
+        public async Task<bool> IsInitializedAsync() => m_initializationResult.IsValueCreated && (await m_initializationResult.Value).Succeeded;
+
+        /// <summary>
+        /// Retrieves the index sources of the specified repositories and initializes the base addresses
+        /// </summary>
+        /// <remarks>Thread safe. Subsequents initializations have no effect and return the same result as the first one did.</remarks>
+        public Task<Possible<bool>> TryInitAsync() => m_initializationResult.Value;
+
+        /// <summary>
+        /// Tries to retrieve the layout and nuspec of the given package
+        /// </summary>
+        /// <remarks>Thread safe</remarks>
+        public async Task<Possible<NugetInspectedPackage>> TryInspectAsync(INugetPackage identity)
+        {
+            Contract.Assert(await IsInitializedAsync(), "TryInitAsync() must be succesfully called first");
+
+            if (m_packageBaseAddress.Count == 0)
+            {
+                return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository);
+            }
+
+            // Build the URI for the requested package and try to inspect it
+            var packageIdLowerCase = identity.Id.ToLowerInvariant();
+            var version = new NuGet.Versioning.NuGetVersion(identity.Version).ToNormalizedString();
+
+            Possible<NugetInspectedPackage> maybeInspectedPackage = default;
+            foreach (var baseAddress in m_packageBaseAddress)
+            {
+                // URIs for retrieving the nuspec and the nupkg
+                var packageUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.{version}.nupkg";
+                var nuspecUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.nuspec";
+
+                maybeInspectedPackage = await TryInspectPackageAsync(identity, new Uri(packageUri), new Uri(nuspecUri), baseAddress.Value);
+                if (maybeInspectedPackage.Succeeded)
+                {
+                    return maybeInspectedPackage;
+                }
+            }
+
+            return maybeInspectedPackage.Failure;
+        }
+
+        private async Task<Possible<NugetInspectedPackage>> TryInspectPackageAsync(INugetPackage identity, Uri nupkgUri, Uri nuspecUri, PackageSourceCredential packageSourceCredential)
+        {
+            AuthenticationHeaderValue authenticationHeader = null;
+
+            // If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
+            // We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
+            if (packageSourceCredential != null)
+            {
+                authenticationHeader = VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(packageSourceCredential.PasswordText);
+            }
+
+            // We want to be able to read the zip file central directory, where the layout of the package is. This is at the end of a zip file, 
+            // so we'll start requesting partial chunks of the content starting from the end and increasingly request more until we can understand
+            // the zip central directory
+
+            int retries = 3;
+
+            while (true)
+            {
+                try
+                {
+                    using (var httpClient = new HttpClient())
+                    {
+                        httpClient.Timeout = TimeSpan.FromMinutes(1);
+
+                        // Use authentication if defined
+                        if (authenticationHeader != null)
+                        {
+                            httpClient.DefaultRequestHeaders.Accept.Clear();
+                            httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+                            httpClient.DefaultRequestHeaders.Authorization = authenticationHeader;
+                        }
+
+                        // We need the nuspec file for analyzing the package
+                        var response = await httpClient.GetAsync(nuspecUri, m_cancellationToken);
+
+                        if (!response.IsSuccessStatusCode)
+                        {
+                            return NugetFailure.CreateNugetInvocationFailure(identity, response.ReasonPhrase);
+                        }
+
+                        var nuspec = await response.Content.ReadAsStringAsync();
+
+                        // Now inspect the content of the nupkg
+                        return (await InspectContentAsync(identity, nupkgUri, authenticationHeader, httpClient)).Then(content => new NugetInspectedPackage(nuspec, content));
+                    }
+                }
+                catch (Exception e) when (e is HttpRequestException || e is AggregateException || e is TaskCanceledException)
+                {
+                    retries--;
+
+                    if (retries == 0)
+                    {
+                        return NugetFailure.CreateNugetInvocationFailure(identity, e);
+                    }
+                }
+            }
+        }
+
+        private async Task<Possible<IReadOnlyList<RelativePath>>> InspectContentAsync(INugetPackage identity, Uri nupkgUri, AuthenticationHeaderValue authenticationHeader, HttpClient httpClient)
+        {
+            long? chunkStart = null;
+            HttpResponseMessage response;
+            bool forceFullDownload = false;
+            var partialPackage = new MemoryStream();
+
+            try
+            {
+                // How many chunks we downloaded so far for the given package
+                int chunkCount = 0;
+                
+                while (true)
+                {
+                    // Only set a download range if we are not forcing a full download
+                    if (!forceFullDownload)
+                    {
+                        // Set the range header to retrieve a particular range of the content
+                        if (!chunkStart.HasValue)
+                        {
+                            // We don't know the total size yet, this is the first request. Start with MinimalChunkSizeInBytes.
+                            httpClient.DefaultRequestHeaders.Add("Range", "bytes=-" + MinimalChunkSizeInBytes);
+                        }
+                        else
+                        {
+                            // This is not the first time we request a chunk, and that means the content we retrieved son far is not enough to read the zip central directory.
+                            // So we already know where the chunk starts (and ends)
+                            httpClient.DefaultRequestHeaders.Add("Range", $"bytes={chunkStart}-{chunkStart + (chunkCount * IncrementChunkSizeInBytes) - 1}");
+                        }
+                    }
+
+                    // TODO: redirect handling may be needed here
+                    response = await httpClient.GetAsync(nupkgUri, HttpCompletionOption.ResponseHeadersRead, m_cancellationToken);
+
+                    // In the rare case where the initial chunk is bigger than the package, we might get this. Just force full download and retry
+                    if (response.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable)
+                    {
+                        httpClient.DefaultRequestHeaders.Remove("Range");
+                        forceFullDownload = true;
+                        continue;
+                    }
+
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        break;
+                    }
+
+                    // Check whether the service decided to ignore the partial request, and instead downloaded the whole thing
+                    if (response.StatusCode == System.Net.HttpStatusCode.OK)
+                    {
+                        forceFullDownload = true;
+                    }
+
+                    long totalLength = 0;
+                    if (!forceFullDownload)
+                    {
+                        totalLength = response.Content.Headers.ContentRange.Length.Value;
+
+                        if (!chunkStart.HasValue)
+                        {
+                            // We just did the first request, chunk start points to MinimalChunkSizeInBytes from the end
+                            chunkStart = totalLength - MinimalChunkSizeInBytes;
+                        }
+                    }
+#if NET_COREAPP_60
+                    using (var chunk = await response.Content.ReadAsStreamAsync(m_cancellationToken))
+#else
+                    using (var chunk = await response.Content.ReadAsStreamAsync())
+#endif
+                    {
+                        // Unfortunately the .net framework does not support prepending a stream, so we do it manually
+                        // TODO: If this becomes a perf/footprint issue we could write a stream wrapper that knows how to compose streams. But
+                        // the expectation is that we shouldn't need to iterate over the same package for very long until we can read it
+                        partialPackage = await PrependToStreamAsync(chunk, partialPackage);
+                    }
+
+                    // We don't want to get in the business of decoding a zip file. For that we use ZipArchive and check whether we get an exception
+                    // when we read it.
+                    // However, ZipArchive expects a stream with the proper length, since it does some validations about the location of the central
+                    // directory baed on that. To avoid creating a stream of the real size (some packages are GB big), we use a ZeroPaddedStream that
+                    // wraps the original stream but pretends to have the required size
+                    IReadOnlyCollection<ZipArchiveEntry> entries;
+                    try
+                    {
+                        var zf = new ZipArchive(forceFullDownload
+                            ? partialPackage 
+                            : new ZeroLeftPaddedStream(partialPackage, totalLength), ZipArchiveMode.Read);
+                        entries = zf.Entries;
+                    }
+                    catch (InvalidDataException ex)
+                    {
+                        // We downloaded the package in full but we cannot recognize it as a zip
+                        if (forceFullDownload)
+                        {
+                            return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {ex.ToStringDemystified()} ");
+                        }
+
+                        // This check is just a heuristics for the rare case when we already downloaded more than 10% of the package total
+                        // size but we can't still read the zip central directory. At this point we can rather download the whole package
+                        if (partialPackage.Length < totalLength * .1)
+                        {
+                            chunkCount++;
+                            // We were not able to read the package central directory with what we downloaded so far. Request another chunk and
+                            // try again
+                            chunkStart -= chunkCount * IncrementChunkSizeInBytes;
+                            httpClient.DefaultRequestHeaders.Remove("Range");
+
+                            continue;
+                        }
+
+                        httpClient.DefaultRequestHeaders.Remove("Range");
+                        forceFullDownload = true;
+                        continue;
+                    }
+
+                    return entries
+                        .Select(entry => entry.FullName.Contains('%') ? System.Net.WebUtility.UrlDecode(entry.FullName) : entry.FullName)
+                        .Where(entry => PackageHelper.IsPackageFile(entry, PackageSaveMode.Files | PackageSaveMode.Nuspec))
+                        .Select(entry => RelativePath.Create(m_stringTable, entry)).ToArray();
+                }
+            }
+            finally
+            {
+                partialPackage.Dispose();
+            }
+
+            return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {response.RequestMessage} ");
+        }
+
+        private async Task<MemoryStream> PrependToStreamAsync(Stream prefix, MemoryStream stream)
+        {
+            // If the destination stream is empty, just copy the prefix over
+            if (stream.Length == 0)
+            {
+                await prefix.CopyToAsync(stream, m_cancellationToken);
+                return stream;
+            }
+
+            var tempPackage = new MemoryStream();
+            await prefix.CopyToAsync(tempPackage, m_cancellationToken);
+            
+            stream.Position = 0;
+            await stream.CopyToAsync(tempPackage, m_cancellationToken);
+
+            stream.Dispose();
+            
+            return tempPackage;
+        }
+    }
+}

Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs

@@ -14,6 +14,8 @@ using TypeScript.Net.Extensions;
 using TypeScript.Net.Types;
 using static TypeScript.Net.DScript.SyntaxFactory;
 using static BuildXL.FrontEnd.Nuget.SyntaxFactoryEx;
+using BuildXL.FrontEnd.Sdk;
+using BuildXL.FrontEnd.Script.Literals;
 
 namespace BuildXL.FrontEnd.Nuget
 {
@@ -25,22 +27,31 @@ namespace BuildXL.FrontEnd.Nuget
         private readonly PathTable m_pathTable;
         private readonly PackageOnDisk m_packageOnDisk;
         private readonly NugetAnalyzedPackage m_analyzedPackage;
-
+        private readonly IReadOnlyDictionary<string, string> m_repositories;
         private readonly NugetFrameworkMonikers m_nugetFrameworkMonikers;
-
+        private readonly AbsolutePath m_sourceDirectory;
         private readonly PathAtom m_xmlExtension;
         private readonly PathAtom m_pdbExtension;
+        private readonly int? m_timeoutInMinutes;
 
         /// <summary>Current spec generation format version</summary>
-        public const int SpecGenerationFormatVersion = 11;
+        public const int SpecGenerationFormatVersion = 12;
 
         /// <nodoc />
-        public NugetSpecGenerator(PathTable pathTable, NugetAnalyzedPackage analyzedPackage)
+        public NugetSpecGenerator(
+            PathTable pathTable, 
+            NugetAnalyzedPackage analyzedPackage, 
+            IReadOnlyDictionary<string, string> repositories,
+            AbsolutePath sourceDirectory,
+            int? timeoutInMinutes = null)
         {
             m_pathTable = pathTable;
             m_analyzedPackage = analyzedPackage;
+            m_repositories = repositories;
             m_packageOnDisk = analyzedPackage.PackageOnDisk;
             m_nugetFrameworkMonikers = new NugetFrameworkMonikers(pathTable.StringTable);
+            m_sourceDirectory = sourceDirectory;
+            m_timeoutInMinutes = timeoutInMinutes;
 
             m_xmlExtension = PathAtom.Create(pathTable.StringTable, ".xml");
             m_pdbExtension = PathAtom.Create(pathTable.StringTable, ".pdb");
@@ -53,13 +64,13 @@ namespace BuildXL.FrontEnd.Nuget
         /// The generated format is:
         /// [optional] import of managed sdk core
         /// [optional] qualifier declaration
-        /// const packageRoot = d`absolute path to the package roo`;
         /// @@public
-        /// export const contents: StaticDirectory = Transformer.sealDirectory(
-        ///    packageRoot,
-        ///    [
-        ///       f`${packageRoot}/file`,
-        ///    ]);
+        /// export const contents: StaticDirectory = NuGetDownloader.downloadPackage(
+        ///    {
+        ///     id: "package ID",
+        ///     version: "X.XX",
+        ///     ...
+        ///    }
         /// @@public
         /// export const pkg: NugetPackage = {contents ...};
         /// </remarks>
@@ -68,8 +79,8 @@ namespace BuildXL.FrontEnd.Nuget
         {
             var sourceFileBuilder = new SourceFileBuilder();
 
-            // 0. Import {Transformer} from "Sdk.Transformers" to be able to seal directories
-            sourceFileBuilder.Statement(ImportDeclaration(new [] { "Transformer" }, "Sdk.Transformers"));
+            // 0. Import * as NugetDownloader from "BuildXL.Tools.NugetDownloader" to be able to download NuGet packages
+            sourceFileBuilder.Statement(ImportDeclaration("NugetDownloader", "BuildXL.Tools.NugetDownloader"));
 
             // 1. Optional import of managed sdk.
             if (analyzedPackage.IsManagedPackage)
@@ -88,12 +99,7 @@ namespace BuildXL.FrontEnd.Nuget
                     .SemicolonAndBlankLine();
             }
 
-            // 3. Declare a public directory that points to the package root, for convenience reasons
-            sourceFileBuilder
-                .Statement(new VariableDeclarationBuilder().Name("packageRoot").Initializer(PropertyAccess("Contents", "packageRoot")).Build())
-                .SemicolonAndBlankLine();
-
-            // Create a sealed directory declaration with all the package content
+            // Create a seal directory declaration with all the package content
             sourceFileBuilder
                 .Statement(CreatePackageContents())
                 .SemicolonAndBlankLine();
@@ -279,24 +285,59 @@ namespace BuildXL.FrontEnd.Nuget
 
         private IStatement CreatePackageContents()
         {
-            var relativepath = "../../../pkgs/" + m_packageOnDisk.Package.Id + "." + m_packageOnDisk.Package.Version;
+            // Arguments for calling the nuget downloader SDK
+            var downloadCallArgs = new List<(string, IExpression expression)>(4) 
+            {
+                    ("id", new LiteralExpression(m_analyzedPackage.Id)),
+                    ("version", new LiteralExpression(m_analyzedPackage.Version)),
+                    ("downloadDirectory", Identifier("outputDir")),
+                    ("extractedFiles", new ArrayLiteralExpression(m_analyzedPackage.PackageOnDisk.Contents
+                        .Select(relativePath => PathLikeLiteral(InterpolationKind.RelativePathInterpolation, relativePath.ToString(m_pathTable.StringTable, PathFormat.Script))))),
+                    ("repositories", new ArrayLiteralExpression(m_repositories.Select(kvp => new ArrayLiteralExpression(new LiteralExpression(kvp.Key), new LiteralExpression(kvp.Value)))))
+            };
+
+            // If a credential provider was used to inspect the package, pass it as an argument to be able to retrieve it.
+            if (m_analyzedPackage.CredentialProviderPath.IsValid)
+            {
+                // If the credential provider is within the source tree, express it in terms of a mount, so the generated
+                // spec is more resilient to cache hits across machines
+                IExpression path;
+                if (m_sourceDirectory.TryGetRelative(m_pathTable, m_analyzedPackage.CredentialProviderPath, out var relativeCredentialProviderPath))
+                {
+                    path = PathLikeLiteral(
+                        InterpolationKind.FileInterpolation, 
+                        new PropertyAccessExpression(new CallExpression(new PropertyAccessExpression("Context", "getMount"), new LiteralExpression("SourceRoot")), "path") , 
+                        "/" + relativeCredentialProviderPath.ToString(m_pathTable.StringTable, PathFormat.Script));
+                }
+                else
+                {
+                    path = PathLikeLiteral(InterpolationKind.FileInterpolation, m_analyzedPackage.CredentialProviderPath.ToString(m_pathTable, PathFormat.Script));
+                }
+                
+                downloadCallArgs.Add(("credentialProviderPath", path));
+            }
+
+            if (m_timeoutInMinutes != null)
+            {
+                downloadCallArgs.Add(("timeoutInMinutes", new LiteralExpression(m_timeoutInMinutes.Value)));
+            }
 
             return new ModuleDeclaration(
                 "Contents",
 
                 Qualifier(new TypeLiteralNode()),
-                PathLikeConstVariableDeclaration("packageRoot", InterpolationKind.DirectoryInterpolation, relativepath, Visibility.Export),
+
+                new VariableDeclarationBuilder().Name("outputDir").Visibility(Visibility.None).Type(new TypeReferenceNode("Directory")).Initializer(
+                    new CallExpression(new PropertyAccessExpression("Context", "getNewOutputDirectory"), new LiteralExpression("nuget"))).Build(),
+                
                 new VariableDeclarationBuilder()
                     .Name("all")
                     .Visibility(Visibility.Public)
                     .Type(new TypeReferenceNode("StaticDirectory"))
                     .Initializer(
-                        new CallExpression(
-                            new PropertyAccessExpression("Transformer", "sealDirectory"),
-                            new Identifier("packageRoot"),
-                            new ArrayLiteralExpression(
-                                m_packageOnDisk.Contents.OrderBy(path => path.ToString(m_pathTable.StringTable)).Select(GetFileExpressionForPath)
-                                    .ToArray())))
+                                new CallExpression(
+                                        new PropertyAccessExpression("NugetDownloader", "downloadPackage"),
+                                        ObjectLiteral(downloadCallArgs.ToArray())))
                     .Build()
             );
         }
@@ -377,11 +418,10 @@ namespace BuildXL.FrontEnd.Nuget
 
         private IExpression GetFileExpressionForPath(RelativePath relativePath)
         {
-            // f`{packageRoot}/relativePath`
-            return PathLikeLiteral(
-                InterpolationKind.FileInterpolation,
-                Identifier("packageRoot"),
-                "/" + relativePath.ToString(m_pathTable.StringTable, PathFormat.Script));
+            // all.assertExistence(r`relativePath`)
+            return new CallExpression(new PropertyAccessExpression("Contents", "all", "getFile"), PathLikeLiteral(
+                InterpolationKind.RelativePathInterpolation,
+                relativePath.ToString(m_pathTable.StringTable, PathFormat.Script)));
         }
 
         private IExpression CreateSimpleBinary(RelativePath binaryFile)

Public/Src/FrontEnd/Nuget/PackageOnDisk.cs

@@ -68,7 +68,9 @@ namespace BuildXL.FrontEnd.Nuget
             out AbsolutePath nuSpecFile,
             out AbsolutePath moduleConfigFile)
         {
-            nuSpecFile = AbsolutePath.Invalid;
+            // The nuspec file is always at the root of the package.
+            nuSpecFile = packageDownloadResult.TargetLocation.Combine(pathTable, nuspecFileName);
+            
             moduleConfigFile = AbsolutePath.Invalid;
 
             foreach (var relativePath in packageDownloadResult.Contents)
@@ -77,11 +79,7 @@ namespace BuildXL.FrontEnd.Nuget
                 if (!relativePath.IsEmpty)
                 {
                     var fileName = relativePath.GetName();
-                    if (fileName.CaseInsensitiveEquals(pathTable.StringTable, nuspecFileName))
-                    {
-                        nuSpecFile = packageDownloadResult.TargetLocation.Combine(pathTable, relativePath);
-                    }
-                    else if (IsModuleConfigFileName(fileName, pathTable.StringTable))
+                    if (IsModuleConfigFileName(fileName, pathTable.StringTable))
                     {
                         moduleConfigFile = packageDownloadResult.TargetLocation.Combine(pathTable, relativePath);
                     }

Public/Src/FrontEnd/Nuget/Tracing/Log.cs

@@ -23,33 +23,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
         /// </summary>
         public static Logger Log { get; } = new LoggerImpl();
 
-        [GeneratedEvent(
-            (ushort)LogEventId.LaunchingNugetExe,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Verbose,
-            Keywords = (int)Keywords.Diagnostics,
-            EventTask = (ushort)Tasks.Parser,
-            Message = "Package nuget://{id}/{version} is being restored by launching nuget.exe with commandline: {commandline}")]
-        public abstract void LaunchingNugetExe(LoggingContext context, string id, string version, string commandline);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.CredentialProviderRequiresToolUrl,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError),
-            EventTask = (ushort)Tasks.Parser,
-            Message = "CredentialProviders for the Nuget resolver are required to specify ToolUrl. '{toolName}' does not do so.")]
-        public abstract void CredentialProviderRequiresToolUrl(LoggingContext context, string toolName);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetDownloadInvalidHash,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Warning,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message = "Configured downloadHash '{hash}' for tool '{toolName}' is invalid. Attempt to download tool without hash guard.")]
-        public abstract void NugetDownloadInvalidHash(LoggingContext context, string toolName, string hash);
-
         [GeneratedEvent(
             (ushort)LogEventId.NugetFailedToCleanTargetFolder,
             EventGenerators = EventGenerators.LocalOnly,
@@ -60,74 +33,13 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
         public abstract void NugetFailedToCleanTargetFolder(LoggingContext context, string id, string version, string targetLocation, string message);
 
         [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedWithNonZeroExitCode,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details.")]
-        public abstract void NugetFailedWithNonZeroExitCode(LoggingContext context, string id, string version, int exitcode, string output);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedDueToSomeWellKnownIssue,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be restored. {message}\r\nSee the buildxl log for more details.")]
-        public abstract void NugetFailedDueToSomeWellKnownIssue(LoggingContext context, string id, string version, string message);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedWithNonZeroExitCodeDetailed,
+            (ushort)LogEventId.NugetInspectionInitialization,
             EventGenerators = EventGenerators.LocalOnly,
             EventLevel = Level.Verbose,
             Keywords = (ushort)Keywords.UserMessage,
             EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. The standard output was:\r\n{stdOut}\r\n. The standard Error was:\r\n{stdErr}")]
-        public abstract void NugetFailedWithNonZeroExitCodeDetailed(
-            LoggingContext context,
-            string id,
-            string version,
-            int exitcode,
-            string stdOut,
-            string stdErr);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedToListPackageContents,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be downloaded because we could not list the content of the folder '{packageFolder}': {message}")]
-        public abstract void NugetFailedToListPackageContents(LoggingContext context, string id, string version, string packageFolder, string message);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedToWriteConfigurationFile,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message = "Could not be write configuration file to '{configFile}': {message}")]
-        public abstract void NugetFailedToWriteConfigurationFile(LoggingContext context, string configFile, string message);
-
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetFailedToWriteConfigurationFileForPackage,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Error,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be processed because we could not write configuration file to '{configFile}': {message}")]
-        public abstract void NugetFailedToWriteConfigurationFileForPackage(
-            LoggingContext context,
-            string id,
-            string version,
-            string configFile,
-            string message);
+            Message = "Nuget inspection info: {message}")]
+        public abstract void NuGetInspectionInitializationInfo(LoggingContext context, string message);
 
         [GeneratedEvent(
             (ushort)LogEventId.NugetFailedToWriteSpecFileForPackage,
@@ -169,21 +81,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
                 "Package nuget://{id}/{version} could not be processed because the nuspec file cannot be found at '{expectedPath}'.")]
         public abstract void NugetFailedNuSpecFileNotFound(LoggingContext context, string id, string version, string expectedPath);
 
-        [GeneratedEvent(
-        (ushort)LogEventId.NugetFailedWithInvalidNuSpecXml,
-        EventGenerators = EventGenerators.LocalOnly,
-        EventLevel = Level.Error,
-        Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError),
-        EventTask = (ushort)Tasks.Parser,
-        Message =
-            "Package nuget://{id}/{version} could not be processed because the nuspec file at: '{nuspecFile}' has unexpected xml contents: {illegalXmlElement}")]
-        public abstract void NugetFailedWithInvalidNuSpecXml(
-                    LoggingContext context,
-                    string id,
-                    string version,
-                    string nuspecFile,
-                    string illegalXmlElement);
-
         [GeneratedEvent(
             (ushort)LogEventId.NugetPackageVersionIsInvalid,
             EventGenerators = EventGenerators.LocalOnly,
@@ -193,16 +90,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
             Message = "Invalid nuget version '{version}' found in config.dsc for package '{packageName}'. Expected version format is 'A.B.C.D'.")]
         public abstract void ConfigNugetPackageVersionIsInvalid(LoggingContext context, string version, string packageName);
 
-        [GeneratedEvent(
-            (ushort)LogEventId.NugetLaunchFailed,
-            EventGenerators = EventGenerators.LocalOnly,
-            EventLevel = Level.Verbose,
-            Keywords = (ushort)Keywords.UserMessage,
-            EventTask = (ushort)Tasks.Parser,
-            Message =
-                "Package nuget://{id}/{version} could not be downloaded because nuget.exe could not be launched: {message}")]
-        public abstract void NugetLaunchFailed(LoggingContext context, string id, string version, string message);
-
         [GeneratedEvent(
             (ushort)LogEventId.NugetStatistics,
             EventGenerators = EventGenerators.LocalOnly,
@@ -218,7 +105,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
             EventGenerators = EventGenerators.LocalOnly,
             EventLevel = Level.Informational,
             EventTask = (ushort)Tasks.Parser,
-            Message = "Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).",
+            Message = "Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).",
             Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
         public abstract void NugetPackageDownloadedCount(LoggingContext context, long packagesDownloadedCount, long totalPackagesCount);
 
@@ -227,7 +114,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
             EventGenerators = EventGenerators.LocalOnly,
             EventLevel = Level.Informational,
             EventTask = (ushort)Tasks.Parser,
-            Message = "Restored {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.",
+            Message = "Inspected {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.",
             Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
         public abstract void NugetPackagesAreRestored(LoggingContext context, long totalPackagesCount, long totalMilliseconds);
 
@@ -254,7 +141,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
             EventGenerators = EventGenerators.LocalOnly,
             EventLevel = Level.Verbose,
             EventTask = (ushort)Tasks.Parser,
-            Message = "Package nuget://{id}/{version} could not be restored because of an unhandled error '{error}'.",
+            Message = "Package nuget://{id}/{version} could not be inspected because of an unhandled error '{error}'.",
             Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
         public abstract void NugetUnhandledError(LoggingContext context, string id, string version, string error);
 
@@ -264,7 +151,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
             EventLevel = Level.Informational,
             EventTask = (ushort)Tasks.Parser,
             Message =
-                "Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}",
+                "Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}",
             Keywords = (int)Keywords.UserMessage | (int)Keywords.OverwritableOnly)]
         public abstract void NugetPackageDownloadedCountWithDetails(LoggingContext context, long packagesDownloadedCount,
             long totalPackagesCount, string packagesToDownloadDetail);

Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs

@@ -14,40 +14,41 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
         None = 0,
 
         // reserved 11300 .. 11400 for nuget
-        LaunchingNugetExe = 11300,
-        CredentialProviderRequiresToolUrl,
-        NugetDownloadInvalidHash,
-        NugetFailedToCleanTargetFolder,
-        NugetFailedWithNonZeroExitCode,
-        NugetFailedWithNonZeroExitCodeDetailed,
-        NugetFailedToListPackageContents,
-        NugetFailedToWriteConfigurationFile,
-        NugetFailedToWriteConfigurationFileForPackage,
-        NugetFailedToWriteSpecFileForPackage,
-        NugetFailedToReadNuSpecFile,
-        NugetFailedWithInvalidNuSpecXml,
-        NugetPackageVersionIsInvalid,
-        NugetLaunchFailed,
-        NugetStatistics,
-        NugetPackagesRestoredCount,
-        NugetPackagesAreRestored,
-        NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen,
-        NugetDependencyVersionWasPickedWithinRange,
-        NugetDependencyVersionDoesNotMatch,
-        NugetUnknownFramework,
-        NugetPackageDownloadDetails,
-        NugetFailedNuSpecFileNotFound,
-        NugetFailedDueToSomeWellKnownIssue,
-        NugetRegenerateNugetSpecs,
-        NugetRegeneratingNugetSpecs,
-        NugetUnhandledError,
-        ForcePopulateTheCacheOptionWasSpecified,
-        NugetConcurrencyLevel,
-        UsePackagesFromDisOptionWasSpecified,
-        NugetFailedDownloadPackagesAndGenerateSpecs,
-        NugetFailedDownloadPackage,
-        NugetFailedGenerationResultFromDownloadedPackage,
-        NugetFailedToWriteGeneratedSpecStateFile,
-        NugetCannotReuseSpecOnDisk,
+        // Reserved LaunchingNugetExe = 11300,
+        // Reserved CredentialProviderRequiresToolUrl = 11301,
+        // Reserved NugetDownloadInvalidHash = 11302,
+        NugetFailedToCleanTargetFolder = 11303,
+        // Reserved NugetFailedWithNonZeroExitCode = 11304,
+        // Reserved NugetFailedWithNonZeroExitCodeDetailed = 11305,
+        // Reserved NugetFailedToListPackageContents = 11306,
+        // Reserved NugetFailedToWriteConfigurationFile = 11307,
+        // Reserved NugetFailedToWriteConfigurationFileForPackage = 11308,
+        NugetFailedToWriteSpecFileForPackage = 11309,
+        NugetFailedToReadNuSpecFile = 11310,
+        // Reserved NugetFailedWithInvalidNuSpecXml = 11311,
+        NugetPackageVersionIsInvalid = 11312,
+        // Reserved NugetLaunchFailed = 11313,
+        NugetStatistics = 11314,
+        NugetPackagesRestoredCount = 11315,
+        NugetPackagesAreRestored = 11316,
+        NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen = 11317,
+        NugetDependencyVersionWasPickedWithinRange = 11318,
+        NugetDependencyVersionDoesNotMatch = 11319,
+        NugetUnknownFramework = 11320,
+        NugetPackageDownloadDetails = 11321,
+        NugetFailedNuSpecFileNotFound = 11322,
+        // Reserved NugetFailedDueToSomeWellKnownIssue = 11323,
+        NugetRegenerateNugetSpecs = 11324,
+        NugetRegeneratingNugetSpecs = 11325,
+        NugetUnhandledError = 11326,
+        ForcePopulateTheCacheOptionWasSpecified = 11327,
+        NugetConcurrencyLevel = 11328,
+        UsePackagesFromDisOptionWasSpecified = 11329,
+        NugetFailedDownloadPackagesAndGenerateSpecs = 11330,
+        NugetFailedDownloadPackage = 11331,
+        NugetFailedGenerationResultFromDownloadedPackage = 11332,
+        NugetFailedToWriteGeneratedSpecStateFile = 11333,
+        NugetCannotReuseSpecOnDisk = 11334,
+        NugetInspectionInitialization = 11335,
     }
 }

Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs

@@ -181,6 +181,8 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
         private SymbolAtom m_semaphoreInfoLimit;
         private SymbolAtom m_semaphoreInfoName;
         private SymbolAtom m_semaphoreInfoIncrementBy;
+        // Explicitly not exposed in DScript since bypassing the salts is not officially supported, and only used for internally scheduled pips
+        private SymbolAtom m_unsafeBypassFingerprintSalt;
 
         private CallSignature ExecuteSignature => CreateSignature(
             required: RequiredParameters(AmbientTypes.ExecuteArgumentsType),
@@ -332,6 +334,7 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
             m_unsafeTrustStaticallyDeclaredAccesses = Symbol("trustStaticallyDeclaredAccesses");
             m_unsafeDisableFullReparsePointResolving = Symbol("disableFullReparsePointResolving");
             m_unsafeDisableSandboxing = Symbol("disableSandboxing");
+            m_unsafeBypassFingerprintSalt = Symbol("bypassFingerprintSalt");
 
             // Semaphore info.
             m_semaphoreInfoLimit = Symbol("limit");
@@ -1416,6 +1419,12 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
             {
                 processBuilder.Options |= Process.Options.DisableSandboxing;
             }
+
+            // UnsafeExecuteArguments.bypassFingerprintSalt
+            if (Converter.ExtractOptionalBoolean(unsafeOptionsObjLit, m_unsafeBypassFingerprintSalt) == true)
+            {
+                processBuilder.Options |= Process.Options.BypassFingerprintSalt;
+            }
         }
 
         private PipId InterpretFinalizationPipArguments(Context context, ObjectLiteral obj)

Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs

@@ -222,6 +222,11 @@ namespace BuildXL.FrontEnd.Sdk
             return EnumerateEntries(path, pattern, recursive, directories: true);
         }
 
+        /// <summary>
+        /// Translates an absolute path according to the defined translations
+        /// </summary>
+        public abstract AbsolutePath Translate(AbsolutePath path);
+
         /// <summary>
         /// A simple IO based file system entry enumeration helper 
         /// </summary>

Public/Src/FrontEnd/Sdk/FrontEndHost.cs

@@ -121,6 +121,7 @@ namespace BuildXL.FrontEnd.Sdk
             string weakPackageFingerprint,
             PackageIdentity package,
             AbsolutePath packageTargetFolder,
+            AbsolutePath pathToNuspec,
             Func<Task<Possible<IReadOnlyList<RelativePath>>>> producePackage);
 
         /// <summary>

Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs

@@ -65,6 +65,12 @@ namespace BuildXL.FrontEnd.Sdk
             };
         }
 
+        /// <inheritdoc/>
+        public override AbsolutePath Translate(AbsolutePath path)
+        {
+            return path;
+        }
+
         /// <summary>
         /// This method is not implemented.
         /// </summary>
@@ -86,7 +92,7 @@ namespace BuildXL.FrontEnd.Sdk
                 return false;
             }
 
-            stream =  m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream;
+            stream = m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream;
 
             return true;
         }

Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs

@@ -42,6 +42,12 @@ namespace BuildXL.FrontEnd.Script.Testing.Helper
             };
         }
 
+        /// <inheritdoc/>
+        public override AbsolutePath Translate(AbsolutePath path)
+        {
+            return path;
+        }
+
         /// <inheritdoc />
         public override bool TryGetFrontEndFile(AbsolutePath path, string frontEnd, out Stream stream)
         {

Public/Src/FrontEnd/UnitTests/Core/DsTest.cs

@@ -30,6 +30,7 @@ using BuildXL.FrontEnd.Workspaces.Core;
 using BuildXL.Native.IO;
 using BuildXL.Pips.Filter;
 using BuildXL.Utilities;
+using BuildXL.Utilities.Collections;
 using BuildXL.Utilities.Configuration;
 using BuildXL.Utilities.Configuration.Mutable;
 using BuildXL.Utilities.Instrumentation.Common;
@@ -1074,6 +1075,8 @@ namespace Test.BuildXL.FrontEnd.Core
             frontEndFactory.InitializeFrontEnds(controller, FrontEndContext, engine.Configuration);
 
             var frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(FrontEndContext.PathTable, engineContext.FileSystem, engine.Configuration);
+            frontEndEngineAbstraction.TryPopulateWithDefaultMountsTable(LoggingContext, engineContext, engine.Configuration, CollectionUtilities.EmptyDictionary<string, string>());
+
             controller.SetState(frontEndEngineAbstraction, GetPipGraph(), config);
 
             var evaluationFilter = fileToProcess.IsValid ? EvaluationFilter.FromSingleSpecPath(FrontEndContext.SymbolTable, FrontEndContext.PathTable, fileToProcess) : EvaluationFilter.Empty;

Public/Src/FrontEnd/UnitTests/Libs/Prelude.Context.ts

@@ -98,7 +98,7 @@ namespace Context {
      */
     interface CurrentHostInformation {
         /** The current Os type */
-        os: "win" | "mac" | "unix";
+        os: "win" | "macOS" | "unix";
         /** The current cpu architecture  */
         cpuArchitecture: "x64" | "x86";
         /** Wheter the current build is run with elevated permissions (admin/sudo) */

Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs

@@ -11,17 +11,18 @@ using Xunit;
 using Xunit.Abstractions;
 using Test.BuildXL.TestUtilities.Xunit;
 using System;
+using BuildXL.Utilities;
 
 namespace Test.BuildXL.FrontEnd.Nuget
 {
     public class NuSpecGeneratorTests
     {
-        private const int CurrentSpecGenVersion = 11;
+        private const int CurrentSpecGenVersion = 12;
 
         private readonly ITestOutputHelper m_output;
         private readonly FrontEndContext m_context;
         private readonly PackageGenerator m_packageGenerator;
-
+        private readonly Dictionary<string, string> m_repositories;
         private static readonly INugetPackage s_myPackage = new NugetPackage() { Id = "MyPkg", Version = "1.99" };
         private static readonly INugetPackage s_systemCollections = new NugetPackage() { Id = "System.Collections", Version = "4.0.11" };
         private static readonly INugetPackage s_systemCollectionsConcurrent = new NugetPackage() { Id = "System.Collections.Concurrent", Version = "4.0.12" };
@@ -42,6 +43,8 @@ namespace Test.BuildXL.FrontEnd.Nuget
 
             var monikers = new NugetFrameworkMonikers(m_context.StringTable);
             m_packageGenerator = new PackageGenerator(m_context, monikers);
+
+            m_repositories = new Dictionary<string, string>() { ["BuildXL"] = "https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json" };
         }
 
         [Fact]
@@ -70,11 +73,11 @@ namespace Test.BuildXL.FrontEnd.Nuget
 </package>",
                 s_packagesOnConfig, new string[] { "lib/net45/my.dll", "lib/net451/my.dll",  "lib/netstandard2.0/my.dll"});
 
-            var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg);
+            var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg);
             var text = spec.ToDisplayStringV2();
             m_output.WriteLine(text);
 
-            string expectedSpec = $@"import {{Transformer}} from ""Sdk.Transformers"";
+            string expectedSpec = $@"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader"";
 import * as Managed from ""Sdk.Managed"";
 
 export declare const qualifier: {{
@@ -82,22 +85,23 @@ export declare const qualifier: {{
     targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"",
 }};
 
-const packageRoot = Contents.packageRoot;
-
 namespace Contents {{
     export declare const qualifier: {{
     }};
-    export const packageRoot = d`../../../pkgs/TestPkg.1.999`;
+    const outputDir: Directory = Context.getNewOutputDirectory(""nuget"");
     @@public
-    export const all: StaticDirectory = Transformer.sealDirectory(
-        packageRoot,
-        [
-            f`${{packageRoot}}/lib/net45/my.dll`,
-            f`${{packageRoot}}/lib/net451/my.dll`,
-            f`${{packageRoot}}/lib/netstandard2.0/my.dll`,
-            f`${{packageRoot}}/TestPkg.nuspec`,
-        ]
-    );
+    export const all: StaticDirectory = NugetDownloader.downloadPackage({{
+        id: ""TestPkg"",
+        version: ""1.999"",
+        downloadDirectory: outputDir,
+        extractedFiles: [
+            r`TestPkg.nuspec`,
+            r`lib/net45/my.dll`,
+            r`lib/net451/my.dll`,
+            r`lib/netstandard2.0/my.dll`,
+        ],
+        repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]],
+    }});
 }}
 
 @@public
@@ -108,9 +112,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
                 ""TestPkg"",
                 ""1.999"",
                 Contents.all,
-                [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)],
-                [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)],
-                [importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg]
+                [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))],
+                [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))],
+                [
+                    importFrom(""System.Collections"").pkg,
+                    importFrom(""System.Collections.Concurrent"").pkg,
+                ]
             );
         case ""net451"":
         case ""net452"":
@@ -122,9 +129,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
                 ""TestPkg"",
                 ""1.999"",
                 Contents.all,
-                [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)],
-                [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)],
-                [importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg]
+                [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))],
+                [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))],
+                [
+                    importFrom(""System.Collections"").pkg,
+                    importFrom(""System.Collections.Concurrent"").pkg,
+                ]
             );
         case ""netstandard2.0"":
         case ""netcoreapp2.0"":
@@ -139,9 +149,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
                 ""TestPkg"",
                 ""1.999"",
                 Contents.all,
-                [Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`)],
+                [Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`))],
                 [
-                    Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`),
+                    Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`)),
                 ],
                 [...addIfLazy(qualifier.targetFramework === ""netstandard2.0"", () => [importFrom(""Newtonsoft.Json"").pkg])]
             );
@@ -150,9 +160,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
     }};
 }}
 )();";
-            XAssert.AreEqual(expectedSpec, text);
+            XAssert.AreEqual(expectedSpec.Trim(), text.Trim());
 
-            const string CurrentSpecHash = "FDBA16F722AB64B5C61F4CBAD40500C9B1944E39";
+            const string CurrentSpecHash = "1291F51E8E59ED9C2F967C01D49CC39FE6DEB9D8";
             ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash);
         }
 
@@ -160,25 +170,29 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
         public void GenerateNuSpecForStub()
         {
             var pkg = m_packageGenerator.AnalyzePackageStub(s_packagesOnConfig);
-            var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg);
+            var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg);
             var text = spec.ToDisplayStringV2();
             m_output.WriteLine(text);
 
-            string expectedSpec = @"import {Transformer} from ""Sdk.Transformers"";
+            string expectedSpec = @"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader"";
 
 export declare const qualifier: {
     targetFramework: ""net10"" | ""net11"" | ""net20"" | ""net35"" | ""net40"" | ""net45"" | ""net451"" | ""net452"" | ""net46"" | ""net461"" | ""net462"" | ""net472"" | ""netstandard1.0"" | ""netstandard1.1"" | ""netstandard1.2"" | ""netstandard1.3"" | ""netstandard1.4"" | ""netstandard1.5"" | ""netstandard1.6"" | ""netstandard2.0"" | ""netcoreapp2.0"" | ""netcoreapp2.1"" | ""netcoreapp2.2"" | ""netcoreapp3.0"" | ""netcoreapp3.1"" | ""net5.0"" | ""net6.0"" | ""netstandard2.1"",
     targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"",
 };
 
-const packageRoot = Contents.packageRoot;
-
 namespace Contents {
     export declare const qualifier: {
     };
-    export const packageRoot = d`../../../pkgs/TestPkgStub.1.999`;
+    const outputDir: Directory = Context.getNewOutputDirectory(""nuget"");
     @@public
-    export const all: StaticDirectory = Transformer.sealDirectory(packageRoot, []);
+    export const all: StaticDirectory = NugetDownloader.downloadPackage({
+        id: ""TestPkgStub"",
+        version: ""1.999"",
+        downloadDirectory: outputDir,
+        extractedFiles: [],
+        repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]],
+    });
 }
 
 @@public
@@ -189,7 +203,7 @@ export const pkg: NugetPackage = {
 };";
             XAssert.ArrayEqual(SplitToLines(expectedSpec), SplitToLines(text));
 
-            const string CurrentSpecHash = "8550BF2B35888E3C0095095F44E2EB06FC2F0576";
+            const string CurrentSpecHash = "389BC336BD4B206394E01B1E76A859E6561CE30D";
             ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash);
         }
 

Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs

@@ -64,7 +64,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
             var nugetResolver = CreateWorkspaceNugetResolverForTesting();
 
             var allPackages = new Dictionary<string, INugetPackage> { [packageOnDisk.Package.Id] = packageOnDisk.Package };
-            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
+            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
 
             XAssert.IsTrue(analyzedPackage.Succeeded);
 
@@ -86,7 +86,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
 
             var allPackages = new Dictionary<string, INugetPackage> { [packageOnDisk.Package.Id] = packageOnDisk.Package };
 
-            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
+            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
 
             XAssert.IsTrue(analyzedPackage.Succeeded);
 
@@ -105,10 +105,10 @@ namespace Test.BuildXL.FrontEnd.Nuget
             var packageText = File.ReadAllText(packageDsc);
             XAssert.IsFalse(packageText.Contains("export * from"));
             XAssert.IsFalse(packageText.Contains("/Foo.Bar.dsc\""));
-            XAssert.IsTrue(packageText.Contains("/pkgs/Foo.Bar.1.2`"));
-            XAssert.IsTrue(packageText.Contains("packageRoot = d`"));
-            XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/a.txt`"));
-            XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/b.txt`"));
+            XAssert.IsTrue(packageText.Contains(@"id: ""Foo.Bar"""));
+            XAssert.IsTrue(packageText.Contains(@"version: ""1.2"""));
+            XAssert.IsTrue(packageText.Contains("r`Folder/a.txt`"));
+            XAssert.IsTrue(packageText.Contains("r`Folder/b.txt`"));
         }
 
         [Fact]
@@ -140,7 +140,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
             var pathTable = m_testContext.PathTable;
             var packageOnDisk = CreateTestPackageOnDisk(includeScriptSpec: false, version: version);
             var nugetResolver = CreateWorkspaceNugetResolverForTesting();
-            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
+            var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
             XAssert.IsTrue(analyzedPackage.Succeeded);
             var allPackages = new Dictionary<string, NugetAnalyzedPackage> { [packageOnDisk.Package.Id] = analyzedPackage.Result };
             XAssert.IsTrue(analyzedPackage.Succeeded);
@@ -314,7 +314,7 @@ $@"module({{
         {
             var allPackages = packagesOnDisk.ToDictionary(kvp => kvp.Package.Id, kvp => kvp.Package);
 
-            var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, false).Result);
+            var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, AbsolutePath.Invalid, false).Result);
 
             resolver.SetDownloadedPackagesForTesting(allAnalyzedPackages);
 

Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs

@@ -44,7 +44,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
                     paths,
                     "testPackageHash"));
 
-            return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false);
+            return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid);
         }
 
         public NugetAnalyzedPackage AnalyzePackageStub(Dictionary<string, INugetPackage> packagesOnConfig)
@@ -58,7 +58,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
                     new PackageIdentity("nuget", nugetPackage.Id, nugetPackage.Version, nugetPackage.Alias),
                     packageFolder: AbsolutePath.Create(m_context.PathTable, X("/X/Pkgs/TestPkgStub/1.999"))));
 
-            return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false);
+            return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid);
         }
     }
 }

Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs

@@ -86,18 +86,6 @@ namespace Test.DScript.Workspaces.Utilities
                 {
                     return new MalformedConfigurationFailure("An array literal is expected for NuGet credential providers");
                 }
-
-                var credentialProviders = expression.Cast<IArrayLiteralExpression>();
-                foreach (var element in credentialProviders.Elements)
-                {
-                    var elementResult = ParseNugetConfigurationFrom(element.As<IObjectLiteralExpression>());
-                    if (!elementResult.Succeeded)
-                    {
-                        return elementResult;
-                    }
-
-                    result.CredentialProviders.Add(elementResult.Result);
-                }
             }
 
             if (configurationExpression.TryFindAssignmentPropertyInitializer(NaiveConfigurationParsingUtilities.ToolUrlFieldName, out expression))

Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs

@@ -145,11 +145,12 @@ namespace BuildXL.Ide.LanguageServer.UnitTests
                 .OrderBy(ci => ci.Label)
                 .ToArray();
 
-            Assert.Equal(1, completionItems.Length);
+            Assert.Equal(2, completionItems.Length);
 
             // TODO: Change to just BuildXL.DScript check after April 15, 2019 when deployed bits have updated.
             Assert.True(completionItems[0].Label == "BuildXL.DScript.LanguageServer.UnitTests.Data.Module" ||
                         completionItems[0].Label == "BuildXLScript.LanguageServer.UnitTests.Data.Module");
+            Assert.True(completionItems[1].Label == "BuildXL.Tools.NugetDownloader");
         }
 
         [Fact]

Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs

@@ -413,9 +413,9 @@ namespace BuildXL.Pips.Graph
         /// <summary>
         /// Add fields to fingerprint
         /// </summary>
-        public void AddFingerprint(IHashingHelper fingerprinter)
+        public void AddFingerprint(IHashingHelper fingerprinter, bool bypassFingerprintSaltAndVersion)
         {
-            if (!string.IsNullOrEmpty(FingerprintSalt))
+            if (!bypassFingerprintSaltAndVersion && !string.IsNullOrEmpty(FingerprintSalt))
             {
                 fingerprinter.Add(nameof(FingerprintSalt), FingerprintSalt);
             }
@@ -455,14 +455,17 @@ namespace BuildXL.Pips.Graph
                 fingerprinter.Add(nameof(ExplicitlyReportDirectoryProbes), 1);
             }
 
-            fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion);
+            if (!bypassFingerprintSaltAndVersion)
+            {
+                fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion);
+            }
         }
 
         private CalculatedFingerprintTuple ComputeWeakFingerprint()
         {
             using (var hasher = new CoreHashingHelper(true))
             {
-                AddFingerprint(hasher);
+                AddFingerprint(hasher, bypassFingerprintSaltAndVersion: false);
                 return new CalculatedFingerprintTuple(hasher.GenerateHash(), hasher.FingerprintInputText);
             }
         }

Public/Src/Pips/Dll/Graph/PipFingerprinter.cs

@@ -173,7 +173,7 @@ namespace BuildXL.Pips.Graph
             Contract.Requires(fingerprinter != null);
             Contract.Requires(pip != null);
 
-            fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp));
+            fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp, pip.BypassFingerprintSalt));
 
             // Fingerprints must change when outputs are hashed with a different algorithm.
             fingerprinter.Add(PipFingerprintField.ContentHashAlgorithmName, s_outputContentHashAlgorithmName);

Public/Src/Pips/Dll/Operations/Pip.cs

@@ -70,6 +70,11 @@ namespace BuildXL.Pips.Operations
         [PipCaching(FingerprintingRole = FingerprintingRole.None)]
         public abstract PipType PipType { get; }
 
+        /// <summary>
+        /// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both EngineEnvironmentSettings.DebugFingerprintSalt and PipFingerprintingVersion.TwoPhaseV2
+        /// </summary>
+        public virtual bool BypassFingerprintSalt => false;
+
         /// <nodoc />
         internal Pip()
         {

Public/Src/Pips/Dll/Operations/Process.Options.cs

@@ -182,6 +182,11 @@ namespace BuildXL.Pips.Operations
             /// Whether to disable sandboxing for this process
             /// </summary>
             DisableSandboxing = 1 << 21,
+
+            /// <summary>
+            /// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both <see cref="EngineEnvironmentSettings.DebugFingerprintSalt"/> and PipFingerprintingVersion.TwoPhaseV2
+            /// </summary>
+            BypassFingerprintSalt = 1 << 22,
         }
     }
 }

Public/Src/Pips/Dll/Operations/Process.cs

@@ -842,6 +842,10 @@ namespace BuildXL.Pips.Operations
         /// </summary>
         public bool HasSharedOpaqueDirectoryOutputs => DirectoryOutputs.Any(d => d.IsSharedOpaque);
 
+        /// <inheritdoc/>
+        [PipCaching(FingerprintingRole = FingerprintingRole.Semantic)]
+        public override bool BypassFingerprintSalt => (ProcessOptions & Options.BypassFingerprintSalt) != 0;
+
         /// <summary>
         /// What policy to apply when merging redirected outputs back
         /// </summary>

Public/Src/Sandbox/MacOs/deployment.dsc

@@ -7,8 +7,6 @@ import * as SdkDeployment from "Sdk.Deployment";
 namespace Deployment {
     export declare const qualifier: {configuration: "debug" | "release"};
 
-    const pkgPath = d`${importFrom("runtime.osx-x64.BuildXL").Contents.all.root}/runtimes/osx-x64/native`;
-
     @@public
     export const macBinaryUsage = Context.getCurrentHost().os === "macOS"
         ? (BuildXLSdk.Flags.isValidatingOsxRuntime ? "package" : "build")
@@ -22,7 +20,9 @@ namespace Deployment {
             contents: macBinaryUsage === "none"
                 ? []
                 : macBinaryUsage === "package"
-                    ? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext`) ]
+                    ? [ SdkDeployment.createFromFilteredStaticDirectory(
+                            importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext`}),
+                            r`.`)]
                     : [{
                         subfolder: a`Contents`,
                         contents: [
@@ -47,7 +47,9 @@ namespace Deployment {
             contents: macBinaryUsage === "none"
                 ? []
                 : macBinaryUsage === "package"
-                    ? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`) ]
+                    ? [ SdkDeployment.createFromFilteredStaticDirectory(
+                        importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`}),
+                        r`.`)]
                     : [{
                         subfolder: a`Contents`,
                         contents: [
@@ -83,30 +85,34 @@ namespace Deployment {
 
     @@public
     export const sandboxMonitor: SdkDeployment.Definition = {
-        contents: [
+        contents: 
             macBinaryUsage === "build"
-                ? Sandbox.monitor
-                : f`${pkgPath}/${qualifier.configuration}/SandboxMonitor`
-        ]
+                ? [Sandbox.monitor]
+                : macBinaryUsage === "package" 
+                    ? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/SandboxMonitor`)]
+                    : []
     };
 
     @@public
     export const interopLibrary: SdkDeployment.Definition = {
         contents: macBinaryUsage === "build"
             ? [ Sandbox.libInterop, Sandbox.libDetours ]
-            : [
-                f`${pkgPath}/${qualifier.configuration}/libBuildXLInterop.dylib`,
-                f`${pkgPath}/${qualifier.configuration}/libBuildXLDetours.dylib`
-            ]
+            : macBinaryUsage === "package" 
+                ? [
+                    importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLInterop.dylib`),
+                    importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLDetours.dylib`)
+                ]
+                : []
     };
 
     @@public
     export const coreDumpTester: SdkDeployment.Definition = {
-        contents: [
+        contents: 
             macBinaryUsage === "build"
-                ? Sandbox.coreDumpTester
-                : f`${pkgPath}/${qualifier.configuration}/CoreDumpTester`
-        ]
+                ? [Sandbox.coreDumpTester]
+                : macBinaryUsage === "package" 
+                    ? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/CoreDumpTester`)]
+                    : []
     };
 
     @@public

Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs

@@ -218,7 +218,7 @@ namespace BuildXL.FrontEnd.Script.Analyzer
                     Contract.AssertNotNull(frontEndEngineAbstraction);
 
                     AddConfigurationMounts(config, mountsTable);
-                    languageServiceEngine.SetMountsTable(mountsTable);
+                    languageServiceEngine.UpdateMountsTable(mountsTable);
                 }
 
                 if (frontEndEngineAbstraction == null)

Public/Src/Tools/FileDownloader/Downloader.cs

@@ -2,21 +2,19 @@
 // Licensed under the MIT License.
 
 using System;
-using System.ComponentModel;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
 using BuildXL.Cache.ContentStore.Hashing;
 using BuildXL.Native.IO;
 using BuildXL.Storage;
 using BuildXL.ToolSupport;
 using BuildXL.Utilities;
-using Microsoft.IdentityModel.Clients.ActiveDirectory;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
+using BuildXL.Utilities.VstsAuthentication;
 
 namespace Tool.Download
 {
@@ -94,11 +92,15 @@ namespace Tool.Download
 
                     var httpRequest = new HttpRequestMessage(HttpMethod.Get, arguments.Url);
 
+                    var logger = new StringBuilder();
+
                     // If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
                     // We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
-                    if (IsVSTSPackageSecureURI(arguments.Url) &&
-                        await TryGetAuthenticationHeaderAsync(arguments.Url) is var authHeader &&
-                        authHeader != null)
+                    if (VSTSAuthenticationHelper.IsVSTSPackageSecureURI(arguments.Url) &&
+                        await VSTSAuthenticationHelper.IsAuthenticationRequiredAsync(arguments.Url, CancellationToken.None, logger) &&
+                        await VSTSAuthenticationHelper.TryGetAuthenticationCredentialsAsync(arguments.Url, CancellationToken.None) is var maybeAuthCredentials &&
+                        maybeAuthCredentials.Succeeded &&
+                        VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(maybeAuthCredentials.Result.pat) is var authHeader)
                     {
                         httpRequest.Headers.Accept.Clear();
                         httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
@@ -162,175 +164,5 @@ namespace Tool.Download
             return await ContentHashingUtilities.HashContentStreamAsync(fs, hashType);
 
         }
-
-        private bool IsVSTSPackageSecureURI(Uri downloadURI)
-        {
-            return downloadURI.Scheme == "https" && 
-                (downloadURI.Host.EndsWith(".pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith(".pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase)) ;
-        }
-
-        /// <summary>
-        /// Tries to authenticate using a credential provider and fallback to IWA if that fails.
-        /// </summary>
-        private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderAsync(Uri uri)
-        {
-            var result = await TryGetAuthenticationHeaderValueWithCredentialProviderAsync(uri);
-            if (result != null)
-            {
-                return result;
-            }
-
-            return await TryGetAuthenticationHeaderWithIWAAsync(uri);
-        }
-
-        /// <summary>
-        /// Tries to get an authentication token using Integrated Windows Authentication with the current logged in user
-        /// </summary>
-        /// <returns>Null if authentication fails</returns>
-        /// <remarks>
-        /// The auth token is acquired to address simple auth cases for retrieving packages from VSTS feeds from Windows domain
-        /// joined machines where IWA is enabled.
-        /// See https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/wiki/Integrated-Windows-Authentication
-        /// </remarks>
-        private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderWithIWAAsync(Uri uri)
-        {
-            var authenticationContext = new AuthenticationContext(m_authority);
-
-            try
-            {
-                var userCredential = new UserCredential($"{Environment.UserName}@{Tenant}");
-
-                // Many times the user UPN cannot be automatically retrieved, so we build it based on the current username and tenant. This might
-                // not be perfect but should work for most cases. Getting the UPN for a given user from AD requires authentication as well.
-                var result = await authenticationContext.AcquireTokenAsync(Resource, Client, userCredential); ;
-                return new AuthenticationHeaderValue("Bearer", result.AccessToken);
-            }
-            catch (AdalException ex)
-            {
-                Console.WriteLine($"Download resolver was not able to authenticate using Integrated Windows Authentication for '{uri}': {ex.Message}");
-                // Getting an auth token via IWA is on a best effort basis. If anything fails, we silently continue without auth
-                return null;
-            }
-        }
-
-        /// <summary>
-        /// Tries to get an authentication token using a credential provider
-        /// </summary>
-        /// <returns>Null if authentication fails</returns>
-        /// <remarks>
-        /// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers
-        /// using an environment variable.
-        /// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general
-        /// </remarks>
-        private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderValueWithCredentialProviderAsync(Uri uri)
-        {
-            string credentialProviderPath = DiscoverCredentialProvider(uri);
-
-            if (credentialProviderPath == null)
-            {
-                // Failure has been logged already
-                return null;
-            }
-
-            // Call the provider with the requested URI in non-interactive mode.
-            var processInfo = new ProcessStartInfo
-            {
-                FileName = credentialProviderPath,
-                Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive",
-                RedirectStandardOutput = true,
-                CreateNoWindow = true,
-            };
-
-            using (var process = Process.Start(processInfo))
-            {
-                try
-                {
-                    #pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
-                    process?.WaitForExit();
-                    #pragma warning restore AsyncFixer02
-                }
-                catch (Exception e) when (e is SystemException || e is Win32Exception)
-                {
-                    ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}");
-                    return null;
-                }
-
-                if (process == null)
-                {
-                    // The process was not started
-                    ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not start credential provider process '{credentialProviderPath}'.");
-                    return null;
-                }
-
-                // Check whether the authorization succeeded
-                if (process.ExitCode == 0)
-                {
-                    try
-                    {
-                        // The response should be a well-formed JSON with a 'password' entry representing the PAT
-                        var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput));
-                        var pat = response.GetValue("Password");
-
-                        if (pat == null)
-                        {
-                            ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'.");
-                            return null;
-                        }
-
-                        // We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT
-                        return new AuthenticationHeaderValue("Basic",
-                            Convert.ToBase64String(
-                                System.Text.ASCIIEncoding.ASCII.GetBytes(
-                                string.Format("{0}:{1}", "", pat))));
-                    }
-                    catch (JsonReaderException)
-                    {
-                        ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' response is not a well-formed JSON.");
-                        return null;
-                    }
-                }
-
-                ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}.");
-            }
-
-            return null;
-        }
-
-        private string DiscoverCredentialProvider(Uri uri)
-        {
-            var paths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH");
-
-            if (paths == null)
-            {
-                ReportAuthenticationViaCredentialProviderFailed(uri, $"NUGET_CREDENTIALPROVIDERS_PATH is not set.");
-                return null;
-            }
-
-            // Here we do something slightly simpler than what NuGet does and just look for the first credential
-            // provider we can find
-            string credentialProviderPath = null;
-            foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
-            {
-                credentialProviderPath = Directory.EnumerateFiles(path, "credentialprovider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault();
-                if (credentialProviderPath != null)
-                {
-                    break;
-                }
-            }
-
-            if (credentialProviderPath == null)
-            {
-                ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider was not found under '{paths}'.");
-
-                return null;
-            }
-
-            return credentialProviderPath;
-        }
-
-        private void ReportAuthenticationViaCredentialProviderFailed(Uri url, string details)
-        {
-            Console.WriteLine($"Download resolver was not able to authenticate using a credential provider for '{url}': {details}") ;
-        }
     }
 }

Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc

@@ -21,6 +21,7 @@ namespace FileDownloader {
         sources: globR(d`.`, "Downloader*.cs"),
         references:[
             importFrom("Newtonsoft.Json").pkg,
+            importFrom("BuildXL.Utilities").VstsAuthentication.dll,
             importFrom("BuildXL.Utilities").Collections.dll,
             importFrom("BuildXL.Utilities").Native.dll,
             importFrom("BuildXL.Utilities.Instrumentation").Common.dll,

Public/Src/Tools/NugetDownloader/App.config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <runtime>
+        <ThrowUnobservedTaskExceptions enabled="true"/>
+    </runtime>
+</configuration>

Public/Src/Tools/NugetDownloader/ConsoleLogger.cs

@@ -0,0 +1,84 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading.Tasks;
+using NuGet.Common;
+
+namespace NugetDownloader
+{
+    /// <summary>
+    /// Logs all messages to the console
+    /// </summary>
+    internal class ConsoleLogger : ILogger
+    {
+        /// <nodoc/>
+        public void Log(LogLevel level, string data)
+        {
+            Console.WriteLine($"[{level}]:{data}");
+        }
+
+        /// <nodoc/>
+        public void Log(ILogMessage message)
+        {
+            Console.WriteLine(message.FormatWithCode());
+        }
+
+        /// <nodoc/>
+        public Task LogAsync(LogLevel level, string data)
+        {
+            Log(level, data);
+            return Task.CompletedTask;
+        }
+
+        /// <nodoc/>
+        public Task LogAsync(ILogMessage message)
+        {
+            Log(message);
+            return Task.CompletedTask;
+
+        }
+
+        /// <nodoc/>
+        public void LogDebug(string data)
+        {
+            Log(LogLevel.Debug, data);
+        }
+
+        /// <nodoc/>
+        public void LogError(string data)
+        {
+            Log(LogLevel.Error, data);
+        }
+
+        /// <nodoc/>
+        public void LogInformation(string data)
+        {
+            Log(LogLevel.Information, data);
+        }
+
+        /// <nodoc/>
+        public void LogInformationSummary(string data)
+        {
+            Log(LogLevel.Information, data);
+        }
+
+        /// <nodoc/>
+        public void LogMinimal(string data)
+        {
+            Log(LogLevel.Information, data);
+        }
+
+        /// <nodoc/>
+        public void LogVerbose(string data)
+        {
+            Log(LogLevel.Verbose, data);
+        }
+
+        /// <nodoc/>
+        public void LogWarning(string data)
+        {
+            Log(LogLevel.Warning, data);
+        }
+    }
+}

Public/Src/Tools/NugetDownloader/LiteralFiles/Tool.NugetDownloader.dsc.literal

@@ -0,0 +1,125 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import {Artifact, Cmd, Transformer, Tool} from "Sdk.Transformers";
+
+export declare const qualifier: {};
+
+const downloaderDefinition: Transformer.ToolDefinition = {
+    // CODESYNC: keep in sync with deployment at Public\Src\FrontEnd\Nuget\BuildXL.FrontEnd.Nuget.dsc
+    exe: Context.isWindowsOS()? f`${Context.getBuildEngineDirectory()}/NugetDownloader.exe` : f`${Context.getBuildEngineDirectory()}/NugetDownloader`,
+    description: "BuildXL NuGet downloader",
+    dependsOnWindowsDirectories: true,
+    dependsOnAppDataDirectory: true,
+    prepareTempDirectory: true,
+    untrackedDirectoryScopes: [
+        d`${Context.getMount("ProgramData").path}/Microsoft/NetFramework/BreadcrumbStore`,
+        ...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("LocalLow").path}/Microsoft/CryptnetUrlCache`]),
+        ...addIfLazy(Context.getCurrentHost().os === "macOS", () => [d`/dev/dtracehelper`, d`/private/var/run/mDNSResponder`, d`/private/var/folders`, d`/dev/tty`, d`/usr/share/zoneinfo`, d`/usr/share/icu`]),
+        d`${Context.getMount("UserProfile").path}/.dotnet/corefx/cryptography`
+    ],
+};
+
+
+/**
+ * Arguments for downloading a NuGet package under a specified directory
+ */
+@@public
+export interface Arguments extends Transformer.RunnerArguments {
+    id: string;
+    version: string;
+    downloadDirectory: Directory;
+    extractedFiles: RelativePath[];
+    repositories: [string, string][];
+    credentialProviderPath?: File;
+    timeoutInMinutes?: number;
+}
+
+/**
+ * Downloads a NuGet package as specified in the arguments and returns the outputs that result of the execution in a static directory
+ */
+@@public
+export function downloadPackage(args: Arguments) : PartialStaticContentDirectory {
+
+    const arguments: Argument[] = [
+        Cmd.option("/id:", args.id),
+        Cmd.option("/version:", args.version),
+        Cmd.option("/downloadDirectory:", Artifact.none(args.downloadDirectory)),
+        Cmd.options("/repositories:", args.repositories.map(kvp => kvp[0] + "=" + kvp[1])),
+    ];
+
+    // NuGet extraction always injects a parent folder with the package id + version as the name
+    const directoryName = `${args.id}.${args.version}`;
+
+    // Build the list of expected outputs
+    const outputs : File[] = args.extractedFiles.map(relativePath => f`${args.downloadDirectory}/${directoryName}/${relativePath}`);
+
+    // In some cases the package is expected to be empty. In that case, don't run anything and just return an empty seal dir
+    if (outputs.length === 0)
+    {
+        return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, []);
+    }
+
+    let tool = downloaderDefinition;
+
+    // The timeout is not available per-pip but at the tool level
+    if (args.timeoutInMinutes !== undefined) {
+        tool = tool.merge<Transformer.ToolDefinition>({
+            timeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000,
+            // We could have a separate argument for this, but for now let's keep it simple
+            warningTimeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000
+        });
+    }
+
+    // Nuget tries to read the user settings config file, and if it is not there, it creates it.
+    // The creation does not behave well with concurrent executions and write locks issues arise.
+    // Let's give each nuget pip a different app data directory
+    const redirectedAppData = Context.getTempDirectory("redirectedAppData");
+
+    const transformerExecuteArgs : Transformer.ExecuteArguments = {
+        description: `Downloading NuGet package '${args.id}.${args.version}'`,
+        tool: tool,
+        arguments: arguments,
+        workingDirectory: args.downloadDirectory,
+        tags: ["nuget", ...(args.tags || [])],
+        environmentVariables: [
+            ...addIf(args.credentialProviderPath !== undefined, {name: "NUGET_CREDENTIALPROVIDERS_PATH", value: args.credentialProviderPath}),
+            ...addIf(Environment.hasVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS"), {name: "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", value: Environment.getStringValue("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS")}),
+            // In case the artifacts credential provider is being used, make sure we don't cache the token, so we avoid dealing with IsRetry
+            {name: "NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED", value: "False"},
+            // On linux/mac the NuGet SDK needs this variable defined. It doesn't really matter where it is pointing to.
+            ...addIf(!Context.isWindowsOS(), {name: "DOTNET_CLI_HOME", value: redirectedAppData}),
+            {name: "AppData", value: redirectedAppData}
+        ],
+        outputs: outputs,
+        tempDirectory: redirectedAppData,
+        unsafe: <Transformer.UnsafeExecuteArguments>{
+            passThroughEnvironmentVariables: [ 
+                "LocalAppData",
+                "QAUTHMATERIALROOT",
+                "__CREDENTIAL_PROVIDER_LOG_DIR",
+                "__CLOUDBUILD_AUTH_HELPER_ROOT__",
+                "__CLOUDBUILD_AUTH_HELPER_CONFIG__",
+            ],
+            requireGlobalDependencies: false,
+            // The only child process this tool spawns is the credential provider process. This process may write log files that are hard to predict upfront.
+            hasUntrackedChildProcesses: true,
+            // This is to avoid a cache miss on a NuGet download due to changes on the nuget download tool (changes in downtream dependencies for the 
+            // most part, e.g. some framework dll that is changed because of some unrelated change in Bxl)
+            // This is a confidence vote since we own the nuget downloader tool. If for any reason the nuget pips need invalidation, that can
+            // be manually triggered by bumping the NuGet spec fingerprint version.
+            untrackedScopes: [Context.getBuildEngineDirectory()],
+            // Make NuGet pips so fingerprint salts (the general fingerprint V2 version and the debug salt passed with /p:BUILDXL_FINGERPRINT_SALT) are not
+            // included in the weak fingerprint. Observe this option is not exposed in DScript, so we are forcing the hand of the type checker with the cast above
+            // to not complain about this extra field (which will be picked up at runtime)
+            bypassFingerprintSalt: true
+        },
+        // This is to avoid DFAs under the credential provider directory, which is an arbitrarily specified directory that would need to
+        // be contained in a statically declared read mount otherwise
+        allowUndeclaredSourceReads: true,
+    };
+
+    const result = Transformer.execute(transformerExecuteArgs);
+
+    return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, result.getOutputFiles());
+}

Public/Src/Tools/NugetDownloader/LiteralFiles/module.config.dsc.literal

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+module({
+    name: "BuildXL.Tools.NugetDownloader"
+});

Public/Src/Tools/NugetDownloader/NugetDownloader.cs

@@ -0,0 +1,149 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using BuildXL.Native.IO;
+using BuildXL.ToolSupport;
+using BuildXL.Utilities;
+using BuildXL.Utilities.VstsAuthentication;
+using NuGet.Common;
+using NuGet.Configuration;
+using NuGet.Packaging;
+using NuGet.Packaging.Signing;
+using NuGet.Protocol.Core.Types;
+using NugetDownloader;
+
+namespace Tool.Download
+{
+    /// <summary>
+    /// Downloads a Nuget into a given directory from a collection of source repositories
+    /// </summary>
+    internal sealed class NugetDownloader : ToolProgram<NugetDownloaderArgs>
+    {
+        private NugetDownloader() : base("NugetDownloader")
+        {
+        }
+
+        /// <nodoc />
+        public static int Main(string[] arguments)
+        {
+            return new NugetDownloader().MainHandler(arguments);
+        }
+
+        /// <inheritdoc />
+        public override bool TryParse(string[] rawArgs, out NugetDownloaderArgs arguments)
+        {
+            try
+            {
+                arguments = new NugetDownloaderArgs(rawArgs);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine(ex.GetLogEventMessage());
+                arguments = null;
+                return false;
+            }
+        }
+
+        /// <inheritdoc />
+        public override int Run(NugetDownloaderArgs arguments)
+        {
+            return TryDownloadNugetToDiskAsync(arguments).GetAwaiter().GetResult();
+        }
+
+        /// <summary>
+        /// Attempts to download and extract a NuGet package to disk.
+        /// </summary>
+        private async Task<int> TryDownloadNugetToDiskAsync(NugetDownloaderArgs arguments)
+        {
+            Console.WriteLine($"Download started for package '{arguments.Id}' and version '{arguments.Version}' to '{arguments.DownloadDirectory}'.");
+
+            var stopwatch = Stopwatch.StartNew();
+
+            SourceCacheContext cache = new SourceCacheContext();
+
+            var authLogger = new StringBuilder();
+            var maybeRepositories = await VSTSAuthenticationHelper.TryCreateSourceRepositories(arguments.Repositories.Select(kvp => (kvp.Key, kvp.Value)), CancellationToken.None, authLogger);
+
+            // Informational data
+            Console.WriteLine(authLogger.ToString());
+
+            if (!maybeRepositories.Succeeded)
+            {
+                Console.Error.WriteLine($"Failed to access the specified repositories: " + maybeRepositories.Failure.Describe());
+                return 1;
+            }
+
+            bool found = false;
+            ILogger logger = new ConsoleLogger();
+            foreach (var sourceRepository in maybeRepositories.Result)
+            {
+                Console.Write($"Finding resource against source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}'. Authentication is on: {sourceRepository.PackageSource.Credentials != null}");
+
+                // TODO: maybe we need a retry here? or define a default retry in the DScript SDK?
+                FindPackageByIdResource resource = await sourceRepository.GetResourceAsync<FindPackageByIdResource>();
+
+                if (resource == null)
+                {
+                    Console.Write($"Resource under source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}' not found.");
+                    continue;
+                }
+
+                var destinationFilePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+
+                // We don't really need to materialize the nupkg file on disk, but a memory stream has a 2GB limitation, and NuGet packages
+                // can be larger than that. Just place it on disk and remove it after we are done extracting.
+                using (var packageStream = new FileStream(
+                    destinationFilePath,
+                    FileMode.Create,
+                    FileAccess.ReadWrite,
+                    FileShare.ReadWrite | FileShare.Delete,
+                    bufferSize: 4096))
+                {
+                    if (await resource.CopyNupkgToStreamAsync(
+                        arguments.Id,
+                        arguments.Version,
+                        packageStream,
+                        cache,
+                        logger,
+                        CancellationToken.None))
+                    {
+                        found = true;
+
+                        await PackageExtractor.ExtractPackageAsync(
+                            sourceRepository.PackageSource.Source,
+                            packageStream,
+                            new PackagePathResolver(arguments.DownloadDirectory),
+                            new PackageExtractionContext(
+                                PackageSaveMode.Files | PackageSaveMode.Nuspec, 
+                                XmlDocFileSaveMode.None, 
+                                ClientPolicyContext.GetClientPolicy(NullSettings.Instance, logger), 
+                                logger),
+                            CancellationToken.None);
+
+                        FileUtilities.DeleteFile(destinationFilePath);
+
+                        break;
+                    }
+                }
+            }
+
+            if (!found)
+            {
+                Console.Error.WriteLine($"Could not find NuGet package '{arguments.Id}' with version '{arguments.Version.ToNormalizedString()}' under any of the provided repositories.");
+                return 1;
+            }
+
+            Console.WriteLine($"Finished downloading package '{arguments.Id}' with version '{arguments.Version}' in {stopwatch.ElapsedMilliseconds}ms");
+
+            return 0;
+        }
+    }
+}

Public/Src/Tools/NugetDownloader/NugetDownloaderArgs.cs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using BuildXL.ToolSupport;
+using NuGet.Versioning;
+
+namespace Tool.Download
+{
+    /// <nodoc/>
+    internal sealed partial class NugetDownloaderArgs : CommandLineUtilities
+    {
+        /// <summary>
+        /// package id of the download
+        /// </summary>
+        public string Id { get; }
+
+        /// <summary>
+        /// package version of the download
+        /// </summary>
+        public NuGetVersion Version { get; }
+
+        /// <summary>
+        /// The directory to place the downloaded file
+        /// </summary>
+        public string DownloadDirectory { get; }
+
+        /// <summary>
+        /// The Nuget feeds (name and Uri) where to download the package from
+        /// </summary>
+        public IReadOnlyDictionary<string, Uri> Repositories { get; }
+
+        /// <nodoc />
+        public NugetDownloaderArgs(string[] args)
+            : base(args)
+        {
+            var repositories = new Dictionary<string, Uri>();
+
+            foreach (Option opt in Options)
+            {
+                if (opt.Name.Equals("id", StringComparison.OrdinalIgnoreCase))
+                {
+                    Id = opt.Value;
+                }
+                else if (opt.Name.Equals("version", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (!NuGetVersion.TryParse(opt.Value, out NuGetVersion version))
+                    {
+                        throw Error($"Malformed version: {opt.Value}.");
+                    }
+
+                    Version = version;
+                }
+                else if (opt.Name.Equals("downloadDirectory", StringComparison.OrdinalIgnoreCase))
+                {
+                    DownloadDirectory = opt.Value;
+                }
+                else if (opt.Name.Equals("repositories", StringComparison.OrdinalIgnoreCase))
+                {
+                    var kvp = ParseKeyValuePair(opt);
+                    repositories[kvp.Key] = new Uri(kvp.Value);
+                }
+                else
+                {
+                    throw Error($"Unsupported option: {opt.Name}.");
+                }
+            }
+
+            Repositories = repositories;
+
+            if (Id == null)
+            {
+                throw Error($"Missing mandatory argument 'id'");
+            }
+
+            if (Version == null)
+            {
+                throw Error($"Missing mandatory argument 'version'");
+            }
+
+            if (Repositories.Count == 0)
+            {
+                throw Error($"Missing mandatory argument 'repositories'");
+            }
+
+            if (string.IsNullOrWhiteSpace(DownloadDirectory))
+            {
+                throw Error($"Missing mandatory argument 'DownloadDirectory'");
+            }
+        }
+    }
+}

Public/Src/Tools/NugetDownloader/Tool.NugetDownloader.dsc

@@ -0,0 +1,62 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import * as Managed from "Sdk.Managed";
+import * as BuildXLSdk from "Sdk.BuildXL";
+import {Transformer} from "Sdk.Transformers";
+import * as Deployment from "Sdk.Deployment";
+import * as MSBuild from "Sdk.Selfhost.MSBuild";
+import * as Frameworks from "Sdk.Managed.Frameworks";
+import * as Shared from "Sdk.Managed.Shared";
+
+namespace NugetDownloader {
+
+    export declare const qualifier : BuildXLSdk.DefaultQualifierWithNet472;
+ 
+    // These are the specs that define the nuget downloader SDK. Since this is a bxl-provided tool
+    // and customers don't have direct exposure to it, they are not places under the usual SDK folder, but
+    // deployed as part of bxl binaries
+    @@public
+    export const specDeployment: Deployment.Definition = {
+        contents: [
+            {subfolder: r`Sdk/Sdk.NugetDownloader`, contents: [
+                {file: f`LiteralFiles/Tool.NugetDownloader.dsc.literal`, targetFileName: a`Tool.NugetDownloader.dsc`},
+                {file: f`LiteralFiles/module.config.dsc.literal`, targetFileName: a`module.config.dsc`},]
+            }
+        ]
+    };    
+
+    @@public
+    export const dll = BuildXLSdk.executable({
+        assemblyName: "NugetDownloader",
+        skipDocumentationGeneration: true,
+        skipDefaultReferences: true,
+        sources: globR(d`.`, "*.cs"),
+        references:[
+            ...addIf(BuildXLSdk.isFullFramework,
+                NetFx.Netstandard.dll
+            ),
+            importFrom("BuildXL.Utilities").VstsAuthentication.dll,
+            importFrom("BuildXL.Utilities").ToolSupport.dll,
+            importFrom("BuildXL.Utilities").Native.dll,
+            importFrom("BuildXL.Utilities").dll,
+
+            importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+        ],
+        runtimeContentToSkip:[
+            importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+        ],
+        runtimeContent: [
+            f`App.config`,
+            specDeployment
+        ],
+        deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Collections.Immutable.dll`] }, 
+    });
+}
+
+

Public/Src/Utilities/Configuration/FrontEndConfigurationExtensions.cs

@@ -33,7 +33,7 @@ namespace BuildXL.Utilities.Configuration
         public const bool DefaultForcePopulatePackageCache = false;
 
         /// <nodoc/>
-        public const bool DefaultRespectWeakFingerprint = false;
+        public const bool DefaultRespectWeakFingerprint = true;
 
         /// <nodoc/>
         public const bool DefaultForceGenerateNuGetSpecs = false;

Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs

@@ -11,8 +11,8 @@ namespace BuildXL.Utilities.Configuration
     public partial interface INugetConfiguration : IArtifactLocation
     {
         /// <summary>
-        /// Optional credential helper to use for NuGet
+        /// The download timeout, in minutes, for each NuGet download pip
         /// </summary>
-        IReadOnlyList<IArtifactLocation> CredentialProviders { get; }
+        int? DownloadTimeoutMin { get; }
     }
 }

Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs

@@ -16,6 +16,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
             Repositories = new Dictionary<string, string>();
             Packages = new List<INugetPackage>();
             DoNotEnforceDependencyVersions = false;
+            Configuration = new NugetConfiguration();
         }
 
         /// <nodoc />
@@ -25,7 +26,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
             Contract.Assume(template != null);
             Contract.Assume(pathRemapper != null);
 
-            Configuration = template.Configuration == null ? null : new NugetConfiguration(template.Configuration);
+            Configuration = new NugetConfiguration(template.Configuration);
             Repositories = new Dictionary<string, string>(template.Repositories.Count);
             foreach (var kv in template.Repositories)
             {

Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs

@@ -1,9 +1,6 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-
 namespace BuildXL.Utilities.Configuration.Mutable
 {
     /// <nodoc />
@@ -12,21 +9,17 @@ namespace BuildXL.Utilities.Configuration.Mutable
         /// <nodoc />
         public NugetConfiguration()
         {
-            CredentialProviders = new List<IArtifactLocation>();
+            DownloadTimeoutMin = 20;
         }
 
         /// <nodoc />
         public NugetConfiguration(INugetConfiguration template)
             : base(template)
         {
-            CredentialProviders = new List<IArtifactLocation>(template.CredentialProviders);
+            DownloadTimeoutMin = template.DownloadTimeoutMin;
         }
 
-        /// <nodoc />
-        [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
-        public List<IArtifactLocation> CredentialProviders { get; set; }
-
-        /// <inheritdoc />
-        IReadOnlyList<IArtifactLocation> INugetConfiguration.CredentialProviders => CredentialProviders;
+        /// <inheritdoc/>
+        public int? DownloadTimeoutMin { get; set; }
     }
 }

Public/Src/Utilities/Native/BuildXL.Native.dsc

@@ -51,7 +51,7 @@ namespace Native {
     export const nativeLinux = [
         ...addIfLazy(qualifier.targetRuntime === "linux-x64", () =>
         [
-            ...globR(d`${importFrom("runtime.linux-x64.BuildXL").Contents.all.root}/runtimes/linux-x64/native/${qualifier.configuration}`, "*.so")
+            ...importFrom("runtime.linux-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/linux-x64/native/${qualifier.configuration}`}).getContent()
         ]),
     ];
 

Public/Src/Utilities/UnitTests/Utilities/ZeroLeftPaddedStreamTests.cs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.IO;
+using BuildXL.Utilities;
+using Test.BuildXL.TestUtilities.Xunit;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Test.BuildXL.Utilities
+{
+    public class ZeroLeftPaddedStreamTests : XunitBuildXLTest
+    {
+        public ZeroLeftPaddedStreamTests(ITestOutputHelper output)
+            : base(output) { }
+
+        [Fact]
+        public void TestBasicFunctionality()
+        {
+            var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 });
+            var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10);
+
+            // Total length should be 10, even if the underlying stream is smaller
+            XAssert.AreEqual(10, paddedStream.Length);
+
+            // Check Position works
+            paddedStream.Position = 9;
+            XAssert.AreEqual(9, paddedStream.Position);
+
+            // Read the whole stream. We should get zeros in the first 5 positions
+            paddedStream.Position = 0;
+            XAssert.AreEqual(0, paddedStream.Position);
+            var result = new byte[10];
+            var bytesRead = paddedStream.Read(result, 0, 10);
+            XAssert.AreArraysEqual(new byte[] { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5 }, result, true);
+            XAssert.AreEqual(10, bytesRead);
+
+            // Read a part of the stream offsetting the destination
+            System.Array.Clear(result, 0, 10);
+            paddedStream.Position = 4;
+            bytesRead = paddedStream.Read(result, 1, 3);
+            XAssert.AreArraysEqual(new byte[] { 0, 0, 1, 2, 0, 0, 0, 0, 0, 0 }, result, true);
+            XAssert.AreEqual(3, bytesRead);
+        }
+
+        [Fact]
+        public void TestSeek()
+        {
+            var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 });
+            var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10);
+
+            var result = new byte[2];
+
+            var newPos = paddedStream.Seek(4, SeekOrigin.Begin);
+            XAssert.AreEqual(4, newPos);
+            paddedStream.Read(result, 0, 2);
+            XAssert.AreEqual(0, result[0]);
+            XAssert.AreEqual(1, result[1]);
+
+            newPos = paddedStream.Seek(7, SeekOrigin.Begin);
+            XAssert.AreEqual(7, newPos);
+            paddedStream.Read(result, 0, 2);
+            XAssert.AreEqual(3, result[0]);
+            XAssert.AreEqual(4, result[1]);
+
+            newPos = paddedStream.Seek(-1, SeekOrigin.End);
+            XAssert.AreEqual(9, newPos);
+            paddedStream.Read(result, 0, 1);
+            XAssert.AreEqual(5, result[0]);
+
+            newPos = paddedStream.Seek(-6, SeekOrigin.End);
+            XAssert.AreEqual(4, newPos);
+            paddedStream.Read(result, 0, 2);
+            XAssert.AreEqual(0, result[0]);
+            XAssert.AreEqual(1, result[1]);
+
+            paddedStream.Position = 6;
+            newPos = paddedStream.Seek(-2, SeekOrigin.Current);
+            XAssert.AreEqual(4, newPos);
+            paddedStream.Read(result, 0, 2);
+            XAssert.AreEqual(0, result[0]);
+            XAssert.AreEqual(1, result[1]);
+
+            paddedStream.Position = 4;
+            newPos = paddedStream.Seek(2, SeekOrigin.Current);
+            XAssert.AreEqual(6, newPos);
+            paddedStream.Read(result, 0, 2);
+            XAssert.AreEqual(2, result[0]);
+            XAssert.AreEqual(3, result[1]);
+        }
+    }
+}
+

Public/Src/Utilities/Utilities/ZeroLeftPaddedStream.cs

@@ -0,0 +1,158 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Diagnostics.ContractsLight;
+using System.IO;
+
+namespace BuildXL.Utilities
+{
+    /// <summary>
+    /// Wraps a seekable stream to expose it with a configurable minimum size, left padding with zeros if needed.
+    /// </summary>
+    /// <remarks>
+    /// No extra memory is taken by the padded portion of the stream
+    /// If the underlying stream is modified, then the behavior of this class is non-deterministic
+    /// </remarks>
+    public class ZeroLeftPaddedStream : Stream
+    {
+        private readonly Stream m_stream;
+        private long m_currentPos = 0;
+        private readonly long m_gap;
+
+        /// <nodoc/>
+        public ZeroLeftPaddedStream(Stream stream, long minimumLength = 0)
+        {
+            Contract.RequiresNotNull(stream);
+            Contract.Requires(stream.CanSeek);
+
+            m_stream = stream;
+            MinimumLength = minimumLength;
+            m_gap = Math.Max(0, MinimumLength - m_stream.Length);
+        }
+        
+        /// <summary>
+        /// The miminum legth the stream has
+        /// </summary>
+        public long MinimumLength { set; get; }
+
+        /// <inheritdoc/>
+        public override bool CanRead => m_stream.CanRead;
+
+        /// <inheritdoc/>
+        public override bool CanSeek => m_stream.CanSeek;
+
+        /// <summary>
+        /// Always false
+        /// </summary>
+        public override bool CanWrite => false;
+
+        /// <inheritdoc/>
+        public override long Length => m_stream.Length + m_gap;
+
+        /// <inheritdoc/>
+        public override long Position { 
+            get => m_currentPos;
+            set
+            { 
+                m_stream.Position = Math.Max(0, value - m_gap);
+                m_currentPos = Math.Min(value, Length); 
+            } 
+        }
+
+        /// <inheritdoc/>
+        public override void Flush()
+        {
+            m_stream.Flush();
+        }
+
+        /// <inheritdoc/>
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            int bytesRead = 0;
+
+            if (Position < m_gap)
+            {
+                // Produce the padded area of the stream with 0-bytes
+                long maxPaddedCount = m_gap - Position;
+                int paddedCount = Math.Min(count, maxPaddedCount > int.MaxValue ? int.MaxValue : (int) maxPaddedCount);
+                if (paddedCount > 0)
+                {
+                    Array.Clear(buffer, offset, paddedCount);
+                    bytesRead = paddedCount;
+                }
+
+                // Read the reminder, if any, from the underlying stream
+                if (paddedCount < count)
+                {
+                    bytesRead += m_stream.Read(buffer, offset + paddedCount, count - paddedCount);
+                }
+            }
+            else
+            {
+                // The position is beyond the padded area, just do a regular read on the underlying stream
+                bytesRead = m_stream.Read(buffer, offset, count);
+            }
+
+            Position += count;
+
+            return bytesRead;
+        }
+
+        /// <inheritdoc/>
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            // Compute the origin of the seek in terms of the starting index on the stream
+            long originIndex = origin switch 
+            { 
+                SeekOrigin.Begin => 0, 
+                SeekOrigin.Current => Position, 
+                SeekOrigin.End => Length, 
+                _ => throw new ArgumentOutOfRangeException(nameof(origin), $"Not expected value {origin}") 
+            };
+
+            // This is the final index of the stream
+            long finalIndex = Math.Min(Length, originIndex + offset);
+
+            if (finalIndex < 0)
+            {
+                throw new ArgumentException("Attempt to seek before the beginning of the stream");
+            }
+
+            if (finalIndex >= m_gap)
+            {
+                m_stream.Seek(finalIndex - m_gap, SeekOrigin.Begin);
+            }
+            else
+            {
+                m_stream.Seek(0, SeekOrigin.Begin);
+            }
+
+            Position = finalIndex;
+
+            return Position;
+        }
+
+        /// <summary>
+        /// Always throws
+        /// </summary>
+        /// <remarks>
+        /// Shouldn't be called since this stream is non-writable
+        /// </remarks>
+        public override void SetLength(long value)
+        {
+            throw new NotImplementedException();
+        }
+
+        /// <summary>
+        /// Always throws
+        /// </summary>
+        /// <remarks>
+        /// Shouldn't be called since this stream is non-writable
+        /// </remarks>
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

Public/Src/Utilities/VstsAuthentication/AuthenticationFailure.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+namespace BuildXL.Utilities.VstsAuthentication
+{
+    /// <summary>
+    /// An authentication related failure when dealing with authenticated NuGet feeds
+    /// </summary>
+    public class AuthenticationFailure : Failure
+    {
+        private readonly string m_failure;
+
+        /// <nodoc/>
+        public AuthenticationFailure(string failure)
+        {
+            m_failure = failure;
+        }
+
+        /// <inheritdoc/>
+        public override BuildXLException CreateException()
+        {
+            return new BuildXLException(m_failure);
+        }
+
+        /// <inheritdoc/>
+        public override string Describe()
+        {
+            return m_failure;
+        }
+
+        /// <inheritdoc/>
+        public override BuildXLException Throw()
+        {
+            throw CreateException();
+        }
+    }
+}

Public/Src/Utilities/VstsAuthentication/BuildXL.Utilties.VstsAuthentication.dsc

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace VstsAuthentication {
+    @@public
+    export const dll = BuildXLSdk.library({
+        assemblyName: "BuildXL.Utilities.VstsAuthentication",
+        sources: globR(d`.`, "*.cs"),
+        references: [
+            $.dll,
+            Native.dll,
+            importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
+            ...addIfLazy(BuildXLSdk.isFullFramework, () => [
+                NetFx.System.Net.Http.dll,
+                NetFx.Netstandard.dll,
+            ]),
+        ],
+    });
+}

Public/Src/Utilities/VstsAuthentication/VSTSAuthenticationHelper.cs

@@ -0,0 +1,335 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Net.Http.Headers;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Diagnostics.ContractsLight;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NuGet.Configuration;
+using NuGet.Protocol;
+using NuGet.Protocol.Core.Types;
+using System.Net.Http;
+using BuildXL.Native.IO;
+using System.Text;
+
+namespace BuildXL.Utilities.VstsAuthentication
+{
+    /// <summary>
+    /// Helper to get authentication credentials against VSTS connections
+    /// </summary>
+    public static class VSTSAuthenticationHelper
+    {
+        // Constant value to target Azure DevOps.
+        private const string Resource = "499b84ac-1321-427f-aa17-267ca6975798";
+        // Visual Studio IDE client ID originally provisioned by Azure Tools.
+        private const string Client = "872cd9fa-d31f-45e0-9eab-6e460a02d1f1";
+        // Microsoft tenant
+        private const string Tenant = "microsoft.com";
+        // Microsoft authority
+        private const string Authority = $"https://login.windows.net/microsoft.com";
+
+        /// <summary>
+        /// Creates a collection of <see cref="SourceRepository"/> given a collection of URIs
+        /// </summary>
+        /// <remarks>
+        /// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable
+        /// </remarks>
+        public static Task<Possible<IReadOnlyCollection<SourceRepository>>> TryCreateSourceRepositories(
+           IEnumerable<(string repositoryName, Uri repositoryUri)> repositories,
+           CancellationToken cancellationToken,
+           StringBuilder logger)
+        {
+            return TryCreateSourceRepositories(repositories, TryDiscoverCredentialProvider, cancellationToken, logger);
+        }
+
+        /// <summary>
+        /// Creates a collection of <see cref="SourceRepository"/> given a collection of URIs
+        /// </summary>
+        /// <remarks>
+        /// For the repositories identified as VSTS ones, credential provider authentication is performed
+        /// </remarks>
+        public static async Task<Possible<IReadOnlyCollection<SourceRepository>>> TryCreateSourceRepositories(
+            IEnumerable<(string repositoryName, Uri repositoryUri)> repositories, 
+            Func<Possible<string>> discoverCredentialProvider,
+            CancellationToken cancellationToken,
+            StringBuilder logger)
+        {
+            Contract.RequiresNotNull(repositories);
+
+            var sourceRepositories = new List<SourceRepository>();
+
+            Lazy<Possible<string>> credentialProviderPath = new Lazy<Possible<string>>(discoverCredentialProvider);
+
+            // For each repository, authenticate if we are dealing with a VSTS feed and retrieve the base addresses
+            foreach (var repository in repositories)
+            {
+                Uri uri = repository.repositoryUri;
+                string uriAsString = uri.AbsoluteUri;
+                var packageSource = new PackageSource(uriAsString, repository.repositoryName);
+
+                logger.AppendLine($"Analyzing {uri.AbsoluteUri}");
+                // For now we only support authentication against VSTS feeds. Feeds outside of that will attempt to connect unauthenticated
+                if (IsVSTSPackageSecureURI(uri) && await IsAuthenticationRequiredAsync(uri, cancellationToken, logger))
+                {
+                    logger.AppendLine($"Authentication required for {uri.AbsoluteUri}. Trying to acquire credentials.");
+
+                    if (!credentialProviderPath.Value.Succeeded)
+                    {
+                        return credentialProviderPath.Value.Failure;
+                    }
+
+                    if (await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath.Value.Result, cancellationToken) is var maybeCredentials)
+                    {
+                        var maybePackageSourceCredential = maybeCredentials
+                            .Then(credentials => new PackageSourceCredential(uriAsString, credentials.username, credentials.pat, true, string.Empty));
+                        
+                        if (maybePackageSourceCredential.Succeeded)
+                        {
+                            logger.AppendLine($"Authentication successfully acquired for {uri.AbsoluteUri}");
+
+                            packageSource.Credentials = maybePackageSourceCredential.Result;
+                        }
+                        else
+                        {
+                            logger.AppendLine($"Failed to authenticate {uri.AbsoluteUri}: {maybePackageSourceCredential.Failure.DescribeIncludingInnerFailures()}");
+                            return maybePackageSourceCredential.Failure;
+                        }
+                    }
+                }
+
+                logger.AppendLine($"Authentication not required for {uri.AbsoluteUri}");
+
+                var sourceRepository = Repository.Factory.GetCoreV3(packageSource);
+                sourceRepositories.Add(sourceRepository);
+            }
+
+            return sourceRepositories;
+        }
+
+        /// <summary>
+        /// Whether the provided Uri requires authentication
+        /// </summary>
+        public static async Task<bool> IsAuthenticationRequiredAsync(Uri uri, CancellationToken cancellationToken, StringBuilder logger)
+        {
+            logger.AppendLine($"Checking whether authentication is required for {uri.AbsoluteUri}");
+            try
+            {
+                var handler = new HttpClientHandler()
+                {
+                    AllowAutoRedirect = false
+                };
+
+                using (var httpClient = new HttpClient(handler))
+                {
+                    httpClient.Timeout = TimeSpan.FromSeconds(5);
+
+                    int retries = 3;
+
+                    while (true)
+                    {
+                        try
+                        {
+                            var httpResponse = await httpClient.GetAsync(uri, cancellationToken);
+
+
+                            logger.AppendLine($"Response for {uri.AbsoluteUri} is: {httpResponse.StatusCode}");
+
+                            return (httpResponse.StatusCode == System.Net.HttpStatusCode.Unauthorized || httpResponse.StatusCode == System.Net.HttpStatusCode.Redirect) && httpResponse.Headers.WwwAuthenticate.Any();
+                        }
+                        catch (TaskCanceledException)
+                        {
+                            logger.AppendLine($"Request for {uri.AbsoluteUri} timed out. Retries left: {retries}");
+                            
+                            retries--;
+                            
+                            // The service seems to be unavailable. Let the pipeline deal with the unavailability, since it will likely result in a better error message
+                            if (retries == 0)
+                            {
+                                return false;
+                            }
+                        }
+                    }
+                }
+            }
+            catch (Exception ex) when (ex is HttpRequestException || ex is AggregateException)
+            {
+                logger.AppendLine($"Exception for {uri.AbsoluteUri}: {ex.Message}");
+
+                // The service seems to be have a problem. Let's pretend it does need auth and let the rest of the pipeline deal with the unavailability, since it will likely result in a better error message
+                return true;
+            }
+        }
+
+        /// <summary>
+        /// Whether the URI is pointing to a VSTS URI
+        /// </summary>
+        public static bool IsVSTSPackageSecureURI(Uri downloadURI)
+        {
+            return downloadURI.Scheme == "https" &&
+                (downloadURI.Host.EndsWith("pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith("pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase));
+        }
+
+        /// <summary>
+        /// Returns an HTTP-based header from a PAT, which can be injected in an HTTP header request message
+        /// </summary>
+        public static AuthenticationHeaderValue GetAuthenticationHeaderFromPAT(string pat)
+        {
+            return new AuthenticationHeaderValue("Basic",
+                            Convert.ToBase64String(
+                                System.Text.ASCIIEncoding.ASCII.GetBytes(
+                                string.Format("{0}:{1}", "", pat))));
+        }
+
+        /// <summary>
+        /// <see cref="TryGetAuthenticationCredentialsAsync(Uri, string, CancellationToken, bool)"/>
+        /// </summary>
+        /// <remarks>
+        /// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable
+        /// </remarks>
+        public static Task<Possible<(string username, string pat)>> TryGetAuthenticationCredentialsAsync(
+            Uri uri,
+            CancellationToken cancellationToken)
+        {
+            return TryDiscoverCredentialProvider()
+                .ThenAsync(async credentialProviderPath => await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken));
+        }
+
+        /// <summary>
+        /// Tries to get an authentication token using a credential provider
+        /// </summary>
+        /// <remarks>
+        /// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers
+        /// using an environment variable.
+        /// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general
+        /// </remarks>
+        public static Task<Possible<(string username, string pat)>> TryGetAuthenticationCredentialsAsync(
+            Uri uri, 
+            string credentialProviderPath, 
+            CancellationToken cancellationToken,
+            bool forceJson = false)
+        {
+            // Call the provider with the requested URI in non-interactive mode.
+            // Some credential providers support specifying JSON as the output format. We first try without this option (some other providers will fail if specified)
+            // but if the provider succeeds but we can't recognize the output as a JSON one, we try again with this option set
+            var processInfo = new ProcessStartInfo
+            {
+                FileName = credentialProviderPath,
+                Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive{(forceJson? " -F JSON" : string.Empty)}",
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true,
+            };
+
+            return RunCredentialProvider(uri, credentialProviderPath, processInfo, forceJson, cancellationToken);
+        }
+
+        private static async Task<Possible<(string username, string pat)>> RunCredentialProvider(
+            Uri uri,
+            string credentialProviderPath, 
+            ProcessStartInfo processInfo, 
+            bool jsonWasForced,
+            CancellationToken cancellationToken)
+        {
+            var failureCommon = $"Unable to authenticate using a credential provider for '{uri}':";
+
+            using (var process = Process.Start(processInfo))
+            {
+                try
+                {
+#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
+                    process?.WaitForExit();
+#pragma warning restore AsyncFixer02
+                }
+                catch (Exception e) when (e is SystemException || e is Win32Exception)
+                {
+                    return new AuthenticationFailure($"{failureCommon} Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}");
+                }
+
+                if (process == null)
+                {
+                    // The process was not started
+                    return new AuthenticationFailure($"{failureCommon} Could not start credential provider process '{credentialProviderPath}'.");
+                }
+
+                // Check whether the authorization succeeded
+                if (process.ExitCode == 0)
+                {
+                    try
+                    {
+                        // The response should be a well-formed JSON with a 'password' entry representing the PAT
+                        var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput), cancellationToken);
+                        var pat = response.GetValue("Password");
+
+                        if (pat == null)
+                        {
+                            return new AuthenticationFailure($"{failureCommon} Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'.");
+                        }
+
+                        // We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT
+                        return (response.GetValue("Username")?.ToString(), pat.ToString());
+
+                    }
+                    catch (JsonReaderException)
+                    {
+                        // The JSON is malformed. If we were already forcing JSON, then that's an error. Otherwise, we try again but now forcing JSON
+                        if (jsonWasForced)
+                        {
+                            return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' response is not a well-formed JSON.");
+                        }
+
+                        return await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken, forceJson: true);
+                    }
+                }
+
+                return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}. Details: {await process.StandardError.ReadToEndAsync()}");
+            }
+        }
+
+        private static Possible<string> TryDiscoverCredentialProvider()
+        {
+            var credentialProviderPaths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH");
+
+            if (credentialProviderPaths == null)
+            {
+                return new AuthenticationFailure("Unable to authenticate using a credential provider: NUGET_CREDENTIALPROVIDERS_PATH is not set.");
+            }
+
+            // Here we do something slightly simpler than what NuGet does and just look for the first credential
+            // provider we can find
+            string credentialProviderPath = null;
+            foreach (string path in credentialProviderPaths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
+            {
+                string directory = path;
+
+                // In some cases the entry can point directly to the credential .exe. Let's treat that uniformly and enumerate its parent directory
+                if (FileUtilities.TryProbePathExistence(path, followSymlink: false) is var maybeExistence && 
+                    maybeExistence.Succeeded && 
+                    maybeExistence.Result == PathExistence.ExistsAsFile) 
+                {
+                    directory = Directory.GetParent(path).FullName;
+                }
+                
+                credentialProviderPath = Directory.EnumerateFiles(directory, "CredentialProvider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault();
+                if (credentialProviderPath != null)
+                {
+                    break;
+                }
+            }
+
+            if (credentialProviderPath == null)
+            {
+                return new AuthenticationFailure($"Unable to authenticate using a credential provider: Credential provider was not found under '{credentialProviderPaths}'.");
+            }
+
+            return credentialProviderPath;
+        }
+    }
+}

Shared/Scripts/Init.cmd

@@ -114,6 +114,7 @@ REM *********************************
 	REM We'll conditionally set the credential provider if not set on the machine.
 	REM If not set we will set it to the local one in the enlistment but iwth the b-drive substitution
 	if NOT DEFINED NUGET_CREDENTIALPROVIDERS_PATH (
+		ECHO NUGET_CREDENTIALPROVIDERS_PATH not set. Setting it to %TOOLROOT%
 		set NUGET_CREDENTIALPROVIDERS_PATH=%TOOLROOT%
 	)
 	goto :EOF

Shared/Tools/readme.txt

@@ -0,0 +1,3 @@
+These files are used by:
+- bxl.ps1 in order to retrieve the latest bxl.exe from the corresponding feed and bootstrap the build with the latest LKG. Nuget.exe and the checked-in credential provider are used to authenticate against the VSTS feed. 
+- bxl.ps1 script also sets NUGET_CREDENTIALPROVIDERS_PATH environment variable to point to this folder (if not previously set). That implies that the Nuget resolver, if configured, will also use this credential provider to authenticate.

cg/nuget/cgmanifest.json

@@ -2742,7 +2742,25 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "NuGet.CommandLine",
-          "Version": "4.7.1"
+          "Version": "5.11.0"
+        }
+      }
+    },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "NuGet.Common",
+          "Version": "5.11.0"
+        }
+      }
+    },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "NuGet.Configuration",
+          "Version": "5.11.0"
         }
       }
     },
@@ -2751,7 +2769,25 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "NuGet.Frameworks",
-          "Version": "5.0.0"
+          "Version": "5.11.0"
+        }
+      }
+    },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "NuGet.Packaging",
+          "Version": "5.11.0"
+        }
+      }
+    },
+    {
+      "Component": {
+        "Type": "NuGet",
+        "NuGet": {
+          "Name": "NuGet.Protocol",
+          "Version": "5.11.0"
         }
       }
     },
@@ -2760,7 +2796,7 @@
         "Type": "NuGet",
         "NuGet": {
           "Name": "NuGet.Versioning",
-          "Version": "4.6.0"
+          "Version": "5.11.0"
         }
       }
     },

config.dsc

@@ -38,11 +38,10 @@ config({
             ]
         },
         {
+            // The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH.
             kind: "Nuget",
 
-            // This configuration pins people to a specific version of nuget
-            // The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH.  TODO: It can be alternatively pinned here,
-            // but when it fails to download (e.g. from a share) the build is aborted. Consider making the failure non-blocking.
+            // TODO: remove once the new LKG gets deployed, nuget.exe is not used anymore
             configuration: {
                 toolUrl: "https://dist.nuget.org/win-x86-commandline/v4.9.4/NuGet.exe",
                 hash: "VSO0:17E8C8C0CDCCA3A6D1EE49836847148C4623ACEA5E6E36E10B691DA7FDC4C39200"
@@ -60,10 +59,7 @@ config({
                   }
                 : {
                     "buildxl-selfhost" : "https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL.Selfhost/nuget/v3/index.json",
-                    "nuget.org" : "http://api.nuget.org/v3/index.json",
-                    "roslyn-tools" : "https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json",
-                    "msbuild" : "https://dotnet.myget.org/F/msbuild/api/v3/index.json",
-                    "dotnet-core" : "https://dotnet.myget.org/F/dotnet-core/api/v3/index.json",
+                    "nuget.org" : "https://api.nuget.org/v3/index.json",
                     "dotnet-arcade" : "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json",
                   },
 
@@ -161,9 +157,13 @@ config({
                 { id: "System.Threading.Tasks.Dataflow", version: "4.9.0" },
 
                 // Nuget
-                { id: "NuGet.CommandLine", version: "4.7.1" },
-                { id: "NuGet.Versioning", version: "4.6.0" }, // Can't use the latest becuase nuget extracts to folder with metadata which we don't support yet.
-                { id: "NuGet.Frameworks", version: "5.0.0"}, // needed for qtest on .net core
+                { id: "NuGet.Packaging", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData", "System.Security.Cryptography.Pkcs"] },
+                { id: "NuGet.Configuration", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData"] },
+                { id: "NuGet.Common", version: "5.11.0" },
+                { id: "NuGet.Protocol", version: "5.11.0" },
+                { id: "NuGet.Versioning", version: "5.11.0" }, 
+                { id: "NuGet.CommandLine", version: "5.11.0" },
+                { id: "NuGet.Frameworks", version: "5.11.0"}, // needed for qtest on .net core
 
                 // ProjFS (virtual file system)
                 { id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },