microsoft
/
BuildXL
зеркало из https://github.com/microsoft/BuildXL.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merged PR 659816: Nuget resolver schedules real pips 

This PR changes the NuGet resolver logic so instead of downloading packages at evaluation time, download pips are scheduled so the package download and extraction happens as part of the build. The high-level flow is the following:
- The nuget resolver downloads enough of each NuGet package just to understand its layout (usually <5k) + the nuspec file. This is taking ~10s on my machine. The inspection process is still using the local + bxl cache caching layer, in the same way the fully downloaded packages were using before. So hits are also possible.
- The spec generation process now generates calls into the (internal) nuget downloader SDK. The specs are still generated on disk. We can consider doing this in memory (as other resolvers do), but that can be left for a second iteration.
- A new nuget downloader tool is now deployed as part of bxl binaries, with a small DScript SDK that schedules it. The generated specs use it. Nuget.exe is not used anymore, the nuget api is used instead. This makes the whole process less prone to machine-depending state that nuget.exe reads by default and doesn't allow us to block.

Related work items: #1942615
This commit is contained in:
Serge Mera 2022-05-05 19:02:49 +00:00
Родитель 42fc6f76f1
Коммит eebb9c136a
71 изменённых файлов: 2274 добавлений и 1225 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
Public/Sdk/Experimental/Msvc/Native/Tools/Link/Link.dsc
Просмотреть файл

@ -179,9 +179,9 @@ export function evaluate(args: Arguments): Result {
],
allowedSurvivingChildProcessNames: [
"mspdbsrv.exe",
"VCTIP.exe",
"conhost.exe",
]
],
unsafe: { childProcessesToBreakawayFromSandbox: [a`vctip.exe`] }
});
let midlOutput = <Midl.Result>{

9
Public/Sdk/Public/Managed/Tools/NuGet/empty.config
Просмотреть файл

@ -27,13 +27,16 @@ Each time a new nuget version is updated we must make sure that any configuratio
<solution>
<clear />
</solution>
<packageSourceCredentials>
<clear />
</packageSourceCredentials>
<apikeys>
<clear />
</apikeys>
<activePackageSource>
<clear />
</activePackageSource>
<fallbackPackageFolders>
<clear />
</fallbackPackageFolders>
<packageSourceMapping>
<clear />
</packageSourceMapping>
</configuration>

3
Public/Sdk/Public/Managed/Tools/NuGet/pack.dsc
Просмотреть файл

@ -14,7 +14,8 @@ export const tool : Transformer.ToolDefinition = {
exe: Nuget.Contents.all.getFile(r`tools/NuGet.exe`),
description: "NuGet pack",
untrackedDirectoryScopes: [
d`${Context.getMount("ProgramData").path}/Nuget`
d`${Context.getMount("ProgramData").path}/Nuget`,
...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("ProgramFilesX86").path}/Nuget`]),
],
dependsOnWindowsDirectories: true,
dependsOnAppDataDirectory: true,

6
Public/Sdk/Public/Prelude/Prelude.Configuration.Resolvers.dsc
Просмотреть файл

@ -839,7 +839,11 @@ interface UntrackingSettings {
}
interface NuGetConfiguration extends ToolConfiguration {
credentialProviders?: ToolConfiguration[];
/**
* The download timeout, in minutes, for each NuGet download pip. Defaults to 20m.
* Equivalent to configuring timeoutInMilliseconds for the corresponding Transformers.ToolDefinition
*/
downloadTimeoutMin?: number
}
interface ScriptResolverDefaults {

4
Public/Sdk/SelfHost/BuildXL/Testing/QTest/qtestFramework.dsc
Просмотреть файл

@ -184,9 +184,9 @@ function runTest(args : TestRunArguments) : File[] {
testAssembly: args.testDeployment.primaryFile.path,
qTestType: Qtest.QTestType.msTest_latest,
qTestDirToDeploy: args.testDeployment.contents,
qTestAdapterPath: Transformer.sealDirectory({
qTestAdapterPath: Transformer.sealPartialDirectory({
root: testAdapterPath,
files: globR(testAdapterPath, "*")
files: rootTestAdapterPath.contents.filter(f => f.path.isWithin(testAdapterPath)),
}),
qTestDotNetFramework: getQTestDotNetFramework(),
qTestPlatform: Qtest.QTestPlatform.x64,

22
Public/Src/Engine/Dll/BasicFrontEndEngineAbstraction.cs
Просмотреть файл

@ -23,10 +23,24 @@ namespace BuildXL.Engine
{
}
/// <nodoc />
public void SetMountsTable(MountsTable mountsTable)
/// <summary>
/// Adds all the mounts specified in the given mounts table
/// </summary>
public void UpdateMountsTable(MountsTable mountsTable)
{
m_customMountsTable = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount);
var allMounts = mountsTable.AllMountsSoFar.ToDictionary(mount => mount.Name.ToString(m_pathTable.StringTable), mount => mount);
if (m_customMountsTable == null)
{
m_customMountsTable = allMounts;
}
else
{
foreach (var mount in allMounts)
{
m_customMountsTable[mount.Key] = mount.Value;
}
}
}
/// <summary>
@ -41,7 +55,7 @@ namespace BuildXL.Engine
return false;
}
SetMountsTable(mountsTable);
UpdateMountsTable(mountsTable);
return true;
}

8
Public/Src/Engine/Dll/FrontEndEngineImplementation.cs
Просмотреть файл

@ -93,6 +93,7 @@ namespace BuildXL.Engine
private readonly MountsTable m_mountsTable;
private readonly LoggingContext m_loggingContext;
private readonly DirectoryTranslator m_directoryTranslator;
/// <summary>
/// Creates an instance of <see cref="FrontEndEngineImplementation"/>.
@ -130,6 +131,7 @@ namespace BuildXL.Engine
m_snapshotCollector = snapshotCollector;
GetTimerUpdatePeriod = timerUpdatePeriod;
Layout = configuration.Layout;
m_directoryTranslator = directoryTranslator;
if (ShouldUseSpecCache(configuration))
{
@ -184,6 +186,12 @@ namespace BuildXL.Engine
});
}
/// <inheritdoc/>
public override AbsolutePath Translate(AbsolutePath path)
{
return m_directoryTranslator?.Translate(path, PathTable) ?? path;
}
private bool ShouldUseSpecCache(IConfiguration configuration)
{
var sourceDirectory = configuration.Layout.SourceDirectory.ToString(PathTable);

4
Public/Src/Engine/UnitTests/Engine/BaseEngineTest.cs
Просмотреть файл

@ -197,7 +197,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult
}
protected void AddSdk(string sdkLocation)
protected void AddSdk(string relativeSdkLocation)
{
Configuration.Resolvers.Add(
new SourceResolverSettings
@ -206,7 +206,7 @@ function execute(args: Transformer.ExecuteArguments): Transformer.ExecuteResult
Modules = new List<DiscriminatingUnion<AbsolutePath, IInlineModuleDefinition>>
{
new DiscriminatingUnion<AbsolutePath, IInlineModuleDefinition>(
AbsolutePath.Create(Context.PathTable, sdkLocation)),
AbsolutePath.Create(Context.PathTable, TestDeploymentDir).Combine(Context.PathTable, RelativePath.Create(Context.StringTable, relativeSdkLocation))),
},
});
}

1
Public/Src/Engine/UnitTests/Engine/LazyMaterializationBuildTests.cs
Просмотреть файл

@ -35,7 +35,6 @@ namespace Test.BuildXL.Engine
{
private CacheInitializer m_cacheInitializer;
/* The build used for tests in this class generate the following layout
*
* \HelloWorldFinal.exe.config (source)

36
Public/Src/Engine/UnitTests/Engine/Test.BuildXL.Engine.dsc
Просмотреть файл

@ -3,6 +3,8 @@
import * as DetoursServices from "BuildXL.Sandbox.Windows";
import * as Xml from "Sdk.Xml";
import * as Deployment from "Sdk.Deployment";
import {Transformer} from "Sdk.Transformers";
namespace Engine {
@ -23,8 +25,34 @@ namespace Engine {
},
];
// Update the value of this variable if you change the version of Microsoft.Net.Compilers in config.dsc.
const microsoftNetCompilerSpec = f`${Context.getMount("FrontEnd").path}/Nuget/specs/Microsoft.Net.Compilers/4.0.1/module.config.bm`;
// We generate specs as if the compilers package was source files, but point it to the downloaded NuGet
const compilerSpecsDir = Context.getNewOutputDirectory("compilers-specs");
const microsoftNetCompilerSpec = Transformer.writeAllText(p`${compilerSpecsDir}/module.config.bm`,
"module({name: 'Microsoft.Net.Compilers', version: '4.0.1', nameResolutionSemantics: NameResolutionSemantics.implicitProjectReferences});");
const specFile = Transformer.writeAllLines(p`${compilerSpecsDir}/package.dsc`, [
"import {Transformer} from 'Sdk.Transformers';",
"namespace Contents {",
"export declare const qualifier: {};",
"@@public export const all: StaticDirectory = Transformer.sealPartialDirectory(d`package`, globR(d`package`, '*'));",
"}"
]);
// Deploy the compilers package plus the specs that refer to it
const deployable : Deployment.Definition = {
contents: [
{
subfolder: a`compilers`,
contents: [
{
subfolder: a`package`,
contents: [importFrom('Microsoft.Net.Compilers').Contents.all]
},
specFile,
microsoftNetCompilerSpec
]
}
]
};
@@public
export const categoriesToRunInParallel = [
@ -47,12 +75,11 @@ namespace Engine {
},
parallelGroups: categoriesToRunInParallel,
testRunData: {
MicrosoftNetCompilersSdkLocation: microsoftNetCompilerSpec,
MicrosoftNetCompilersSdkLocation: "compilers/module.config.bm",
},
tools: {
exec: {
dependencies: [
microsoftNetCompilerSpec,
importFrom("Microsoft.Net.Compilers").Contents.all,
importFrom("Microsoft.NETCore.Compilers").Contents.all,
]
@ -87,6 +114,7 @@ namespace Engine {
],
runtimeContent: [
...libsUsedForTesting,
deployable
],
});
}

6
Public/Src/Engine/UnitTests/Scheduler/PipFingerprinterTests.cs
Просмотреть файл

@ -882,6 +882,7 @@ namespace Test.BuildXL.Scheduler
bool preservePathSetCasing = source.Vary(p => p.PreservePathSetCasing);
bool writingToStandardErrorFailsExecution = source.Vary(p => p.WritingToStandardErrorFailsExecution);
bool disableFullReparsePointResolving = source.Vary(p => p.DisableFullReparsePointResolving);
bool bypassFingerprintSalt = source.Vary(p => p.BypassFingerprintSalt);
Process.Options options = Process.Options.None;
if (hasUntrackedChildProcesses)
@ -940,6 +941,11 @@ namespace Test.BuildXL.Scheduler
options |= Process.Options.DisableFullReparsePointResolving;
}
if (bypassFingerprintSalt)
{
options |= Process.Options.BypassFingerprintSalt;
}
return new Process(
executable: executable,
workingDirectory: workingDirectory,

42
Public/Src/FrontEnd/Core/FrontEndHostController.Nuget.cs
Просмотреть файл

@ -52,6 +52,7 @@ namespace BuildXL.FrontEnd.Core
string weakPackageFingerprint,
PackageIdentity package,
AbsolutePath packageTargetFolder,
AbsolutePath pathToNuspec,
Func<Task<Possible<IReadOnlyList<RelativePath>>>> producePackage)
{
// Check if we can reuse a package that is layed out on disk already.
@ -97,6 +98,7 @@ namespace BuildXL.FrontEnd.Core
weakPackageFingerprint,
package,
packageTargetFolder,
pathToNuspec,
possiblePackageContents.Result),
justification: "Okay to ignore putting in cache failure, will happen next time"
);
@ -354,7 +356,7 @@ namespace BuildXL.FrontEnd.Core
// Saving package's hash file on disk.
// But we can do this only when the content is not empty.
var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList();
var packagesContent = packageDescriptor.Contents.Select(k => k.Key).ToList();
if (packagesContent.Count == 0)
{
@ -363,9 +365,6 @@ namespace BuildXL.FrontEnd.Core
return PackageDownloadResult.RecoverableError(package);
}
var newPackageHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, packagesContent);
TryUpdatePackageHashFile(loggingContext, package, packageHashFile, packageHash, newPackageHash);
m_logger.PackageRestoredFromCache(loggingContext, package.GetFriendlyName());
if (forcePopulatePackageCache)
@ -394,11 +393,20 @@ namespace BuildXL.FrontEnd.Core
}
}
// The hash file is stored as part of the package, so we have the package layout available there
var maybePackageHashFile = PackageHashFile.TryReadFrom(GetPackageHashFile(packageTargetFolder));
if (!maybePackageHashFile.Succeeded)
{
m_logger.DownloadPackageFailedDueToCacheError(loggingContext, friendlyName, maybePackageHashFile.Failure.Describe());
return PackageDownloadResult.RecoverableError(package);
}
// Step: Return descriptor indicating it was successfully restored from the cache
return PackageDownloadResult.FromCache(
package,
packageTargetFolder,
packageDescriptor.Contents.Select(c => RelativePath.Create(FrontEndContext.StringTable, c.Key)).ToList(),
maybePackageHashFile.Result.Content.Select(entry => RelativePath.Create(FrontEndContext.StringTable, entry)).ToList(),
weakPackageFingerprintHash.Hash.ToHex());
}
@ -407,6 +415,7 @@ namespace BuildXL.FrontEnd.Core
string weakPackageFingerprint,
PackageIdentity package,
AbsolutePath packageTargetFolder,
AbsolutePath pathToNuspec,
IReadOnlyList<RelativePath> packageContent)
{
var friendlyName = package.GetFriendlyName();
@ -423,11 +432,22 @@ namespace BuildXL.FrontEnd.Core
// Cache was already initialized
var cache = await m_nugetCache;
// Step: Store all the files into the content cache
// Generate the hash file, since that is stored as part of the cache content
var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint);
// The content should have relative paths
var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList();
var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content);
var packageHashFile = GetPackageHashFile(packageTargetFolder);
TryUpdatePackageHashFile(loggingContext, package, packageHashFile, oldHash: null, newHash: newHash);
// Step: Store the nuspec and the hash file into the content cache
var stringKeyedHashes = new List<StringKeyedHash>();
foreach (var relativePath in packageContents)
foreach (var absolutePath in new[] { pathToNuspec, AbsolutePath.Create(PathTable, packageHashFile)})
{
var targetFileLocation = packageTargetFolder.Combine(PathTable, relativePath).Expand(PathTable);
var targetFileLocation = absolutePath.Expand(PathTable);
var result = packageTargetFolder.TryGetRelative(PathTable, absolutePath, out var relativePath);
Contract.Assert(result);
ContentHash contentHash;
try
@ -453,7 +473,6 @@ namespace BuildXL.FrontEnd.Core
}
}
var weakPackageFingerprintHash = cache.GetDownloadFingerprint(weakPackageFingerprint);
// Step: Create a descriptor and store that in the fingerprint store under the weak fingerprint.
var cacheDescriptor = PackageDownloadDescriptor.Create(
friendlyName,
@ -471,11 +490,6 @@ namespace BuildXL.FrontEnd.Core
m_logger.PackageNotFoundInCacheAndDownloaded(loggingContext, package.Id, package.Version, weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint);
// The content should have relative paths
var content = packageContents.Select(rp => rp.ToString(PathTable.StringTable)).ToList();
var newHash = new PackageHashFile(weakPackageFingerprintHash.Hash.ToHex(), weakPackageFingerprint, content);
TryUpdatePackageHashFile(loggingContext, package, GetPackageHashFile(packageTargetFolder), oldHash: null, newHash: newHash);
return Unit.Void;
}

2
Public/Src/FrontEnd/Core/NugetCache.cs
Просмотреть файл

@ -132,7 +132,7 @@ namespace BuildXL.FrontEnd.Core
private static ContentFingerprint CreateDownloadFingerprint(string baseText)
{
// In case something in the cached Bond data becomes incompatible, we must not match.
const string VersionText = ", BondDataVersion=2;FingerprintVersion=1";
const string VersionText = ", BondDataVersion=2;FingerprintVersion=5";
var fingerprint = FingerprintUtilities.Hash(baseText + VersionText);
return new ContentFingerprint(fingerprint);
}

2
Public/Src/FrontEnd/Core/PackageHashFile.cs
Просмотреть файл

@ -31,7 +31,7 @@ namespace BuildXL.FrontEnd.Core
// The file format change will force specs regeneration.
// Change the version if the nuget spec generation has changed in a backward incompatible way.
private const string HashFileFormatVersion = "9";
private const string HashFileFormatVersion = "10";
/// <summary>
/// The minimal number of lines for the hash file.

18
Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc
Просмотреть файл

@ -14,9 +14,11 @@ namespace Nuget {
sources: globR(d`.`, "*.cs"),
references: [
...addIf(BuildXLSdk.isFullFramework,
NetFx.Netstandard.dll,
NetFx.System.IO.Compression.dll,
NetFx.System.Xml.dll,
NetFx.System.Xml.Linq.dll,
NetFx.Netstandard.dll
NetFx.System.Net.Http.dll
),
...addIf(BuildXLSdk.isFullFramework,
importFrom("System.Memory").withQualifier({targetFramework: "netstandard2.0"}).pkg
@ -33,6 +35,7 @@ namespace Nuget {
importFrom("BuildXL.Engine").Processes.dll,
importFrom("BuildXL.Pips").dll,
importFrom("BuildXL.Utilities").dll,
importFrom("BuildXL.Utilities").VstsAuthentication.dll,
importFrom("BuildXL.Utilities").Collections.dll,
importFrom("BuildXL.Utilities").Configuration.dll,
importFrom("BuildXL.Utilities").Interop.dll,
@ -42,12 +45,21 @@ namespace Nuget {
importFrom("BuildXL.Utilities").Script.Constants.dll,
importFrom("Newtonsoft.Json").pkg,
importFrom("NuGet.Versioning").pkg,
importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
...BuildXLSdk.tplPackages,
],
runtimeContent: [
// Keep in sync with path at Public\Sdk\Public\Tools\NugetDownloader\Tool.NugetDownloader.dsc
importFrom("BuildXL.Tools").NugetDownloader.dll
],
internalsVisibleTo: [
"Test.BuildXL.FrontEnd.Nuget"
]
],
});
}

32
Public/Src/FrontEnd/Nuget/Failures/NugeInvocationtFailure.cs Normal file
Просмотреть файл

@ -0,0 +1,32 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using BuildXL.Utilities.Configuration;
using static BuildXL.Utilities.FormattableStringEx;
namespace BuildXL.FrontEnd.Nuget
{
/// <summary>
/// Nuget invocation failed.
/// </summary>
public sealed class NugeInvocationtFailure : NugetFailure
{
private readonly INugetPackage m_package;
private readonly string m_message;
/// <nodoc />
public NugeInvocationtFailure(INugetPackage package, string message)
: base(FailureType.PackageNotFound)
{
m_package = package;
m_message = message?.Trim();
}
/// <inheritdoc />
public override string Describe()
{
return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored. {m_message}.");
}
}
}

38
Public/Src/FrontEnd/Nuget/Failures/NugetFailedWithNonZeroExitCodeFailure.cs
Просмотреть файл

@ -1,38 +0,0 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using BuildXL.Utilities.Configuration;
using static BuildXL.Utilities.FormattableStringEx;
namespace BuildXL.FrontEnd.Nuget
{
/// <summary>
/// Nuget.exe failed with non-zero exit code.
/// </summary>
public sealed class NugetFailedWithNonZeroExitCodeFailure : NugetFailure
{
private readonly INugetPackage m_package;
private readonly int m_exitCode;
private readonly string m_stdOut;
private readonly string m_stdErr;
/// <nodoc />
public NugetFailedWithNonZeroExitCodeFailure(INugetPackage package, int exitCode, string stdOut, string stdErr)
: base(FailureType.PackageNotFound)
{
m_package = package;
m_exitCode = exitCode;
m_stdOut = stdOut?.Trim();
m_stdErr = stdErr?.Trim();
}
/// <inheritdoc />
public override string Describe()
{
var separator = !string.IsNullOrEmpty(m_stdOut) && !string.IsNullOrEmpty(m_stdErr) ? Environment.NewLine : string.Empty;
var output = I($"{m_stdOut}{separator}{m_stdErr}");
return I($"Package nuget://{m_package.Id}/{m_package.Version} could not be restored because nuget.exe failed with exit code '{m_exitCode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details.");
}
}
}

83
Public/Src/FrontEnd/Nuget/Failures/NugetFailure.cs
Просмотреть файл

@ -4,6 +4,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.Linq;
using BuildXL.Utilities;
using BuildXL.Utilities.Configuration;
using TypeScript.Net.Extensions;
@ -21,13 +22,21 @@ namespace BuildXL.FrontEnd.Nuget
public readonly FailureType Type;
/// <nodoc />
public readonly Exception Exception;
public readonly string Message;
/// <nodoc />
public NugetFailure(FailureType failureType, Exception e = null)
{
Type = failureType;
Exception = e;
Message = GetAllMessages(e);
}
/// <nodoc />
public NugetFailure(FailureType failureType, string message)
{
Type = failureType;
Message = message;
}
/// <nodoc />
@ -58,20 +67,28 @@ namespace BuildXL.FrontEnd.Nuget
}
/// <summary>
/// Nuget.exe failed to restore a package.
/// Nuget invocation failure
/// </summary>
public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, int exitCode, string stdOut, string stdErr)
public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, Exception e)
{
Contract.Requires(package != null);
Contract.Requires(exitCode != 0);
return CreateNugetInvocationFailure(package, GetAllMessages(e));
}
/// <summary>
/// Nuget invocation failure
/// </summary>
public static NugetFailure CreateNugetInvocationFailure(INugetPackage package, string message)
{
Contract.RequiresNotNull(package);
Contract.RequiresNotNull(message);
// If the stdOut has the following text: 'NotFound http' or 'WARNING: Unable to find version', it means that the package name or version are not found.
if (stdOut.Contains("NotFound http") || stdOut.Contains("WARNING: Unable to find version"))
if (message.Contains("NotFound http") || message.Contains("WARNING: Unable to find version"))
{
return new CanNotFindPackageFailure(package);
}
return new NugetFailedWithNonZeroExitCodeFailure(package, exitCode, stdOut, stdErr);
return new NugeInvocationtFailure(package, message);
}
/// <summary>
@ -89,10 +106,10 @@ namespace BuildXL.FrontEnd.Nuget
{
if (Package != null)
{
return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Exception?.ToStringDemystified()}");
return I($"Failed to retrieve nuget package '{Package.Id}' version '{Package.Version}' due to {Type.ToString()}. {Message}");
}
return I($"Failed to process nuget packages due to {Type.ToString()}. {Exception?.ToStringDemystified()}");
return I($"Failed to process nuget packages due to {Type.ToString()}. {Message}");
}
/// <inheritdoc />
@ -110,30 +127,15 @@ namespace BuildXL.FrontEnd.Nuget
/// <nodoc />
public enum FailureType
{
/// <nodoc />
FetchNugetExe,
/// <nodoc />
FetchCredentialProvider,
/// <nodoc />
WriteConfigFile,
/// <nodoc />
WriteSpecFile,
/// <nodoc />
CleanTargetFolder,
/// <nodoc />
NugetFailedWithNonZeroExitCode,
/// <nodoc />
NugetFailedWithIoException,
/// <nodoc />
ListPackageContents,
/// <nodoc />
ReadNuSpecFile,
@ -160,6 +162,37 @@ namespace BuildXL.FrontEnd.Nuget
/// <nodoc />
UnhandledError,
/// <nodoc/>
NoBaseAddressForRepository,
}
private static string GetAllMessages(Exception e)
{
if (e is null)
{
return string.Empty;
}
if (e is AggregateException aggregateException)
{
return string.Join(Environment.NewLine, aggregateException.Flatten().InnerExceptions.SelectMany(ie => GetInnerExceptions(ie)).Select(ex => ex.Message));
}
else
{
return string.Join(Environment.NewLine, GetInnerExceptions(e).Select(e => e.Message));
}
}
private static IEnumerable<Exception> GetInnerExceptions(Exception ex)
{
var innerException = ex;
do
{
yield return innerException;
innerException = innerException.InnerException;
}
while (innerException != null);
}
}
}

17
Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs
Просмотреть файл

@ -38,6 +38,14 @@ namespace BuildXL.FrontEnd.Nuget
/// <nodoc />
public MultiValueDictionary<Moniker, INugetPackage> DependenciesPerFramework { get; }
/// <summary>
/// Credential provider path that is used to retrieve the package
/// </summary>
/// <remarks>
/// AbsolutePath.Invalid if none is to be used
/// </remarks>
public AbsolutePath CredentialProviderPath { get; }
/// <nodoc />
public bool IsManagedPackage { get; set; }
@ -129,7 +137,8 @@ namespace BuildXL.FrontEnd.Nuget
NugetFrameworkMonikers nugetFrameworkMonikers,
PackageOnDisk packageOnDisk,
Dictionary<string, INugetPackage> packagesOnConfig,
bool doNotEnforceDependencyVersions)
bool doNotEnforceDependencyVersions,
AbsolutePath credentialProviderPath)
{
m_context = context;
PackageOnDisk = packageOnDisk;
@ -142,6 +151,7 @@ namespace BuildXL.FrontEnd.Nuget
AssemblyToTargetFramework = new MultiValueDictionary<PathAtom, NugetTargetFramework>();
m_dependencies = new List<INugetPackage>();
DependenciesPerFramework = new MultiValueDictionary<PathAtom, INugetPackage>();
CredentialProviderPath = credentialProviderPath;
}
/// <summary>
@ -156,13 +166,14 @@ namespace BuildXL.FrontEnd.Nuget
[CanBeNull] XDocument nuSpec,
PackageOnDisk packageOnDisk,
Dictionary<string, INugetPackage> packagesOnConfig,
bool doNotEnforceDependencyVersions)
bool doNotEnforceDependencyVersions,
AbsolutePath credentialProviderPath)
{
Contract.Requires(context != null);
Contract.Requires(packageOnDisk != null);
var analyzedPackage = new NugetAnalyzedPackage(context, nugetFrameworkMonikers, packageOnDisk,
packagesOnConfig, doNotEnforceDependencyVersions);
packagesOnConfig, doNotEnforceDependencyVersions, credentialProviderPath);
analyzedPackage.ParseManagedSemantics();
if (nuSpec != null && !analyzedPackage.TryParseDependenciesFromNuSpec(nuSpec))

2
Public/Src/FrontEnd/Nuget/NugetFrontEnd.cs
Просмотреть файл

@ -41,7 +41,7 @@ namespace BuildXL.FrontEnd.Nuget
public IReadOnlyCollection<string> SupportedResolvers { get; } = new[] { WorkspaceNugetModuleResolver.NugetResolverName };
/// <inheritdoc />
public bool ShouldRestrictBuildParameters { get; } = true;
public bool ShouldRestrictBuildParameters { get; } = false;
/// <inheritdoc />
public void InitializeFrontEnd(FrontEndHost host, FrontEndContext context, IConfiguration configuration)

31
Public/Src/FrontEnd/Nuget/NugetInspectedPackage.cs Normal file
Просмотреть файл

@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using BuildXL.Utilities;
namespace BuildXL.FrontEnd.Nuget
{
/// <summary>
/// The result of inspecting a package with <see cref="NugetPackageInspector"/>
/// </summary>
public readonly struct NugetInspectedPackage
{
/// <nodoc/>
public readonly string Nuspec { get; }
/// <nodoc/>
public readonly IReadOnlyList<RelativePath> Content { get; }
/// <nodoc/>
public NugetInspectedPackage(string nuspec, IReadOnlyList<RelativePath> content)
{
Contract.RequiresNotNull(nuspec);
Contract.RequiresNotNull(content);
Nuspec = nuspec;
Content = content;
}
}
}

365
Public/Src/FrontEnd/Nuget/NugetPackageInspector.cs Normal file
Просмотреть файл

@ -0,0 +1,365 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics.ContractsLight;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.FrontEnd.Nuget.Tracing;
using BuildXL.Utilities;
using BuildXL.Utilities.Configuration;
using BuildXL.Utilities.Instrumentation.Common;
using BuildXL.Utilities.VstsAuthentication;
using NuGet.Configuration;
using NuGet.Packaging;
using NuGet.Protocol;
namespace BuildXL.FrontEnd.Nuget
{
/// <summary>
/// Retrieves NuGet package layouts and specs from a set of feeds
/// </summary>
/// <remarks>
/// Packages are not necessarily downloaded in full in order to inspect them, and partial downloads are always attempted first
/// </remarks>
public class NugetPackageInspector
{
// A 5K chunk looks reasonable as the initial download: most package central directory fits in 5K, so no extra requests are needed
private const long MinimalChunkSizeInBytes = 5_000;
// On every partial download iteration we linearly grow using a 20K base (i.e. 20, 40, 60, 80, etc.)
private const long IncrementChunkSizeInBytes = 20_000;
private readonly CancellationToken m_cancellationToken;
private readonly LoggingContext m_loggingContext;
private readonly IEnumerable<(string repositoryName, Uri repositoryUri)> m_repositories;
private readonly StringTable m_stringTable;
private readonly Func<Possible<string>> m_discoverCredentialProvider;
private IReadOnlyDictionary<Uri, PackageSourceCredential> m_packageBaseAddress;
private readonly Lazy<Task<Possible<bool>>> m_initializationResult;
/// <nodoc/>
public NugetPackageInspector(
IEnumerable<(string repositoryName, Uri repositoryUri)> repositories,
StringTable stringTable,
Func<Possible<string>> discoverCredentialProvider,
CancellationToken cancellationToken,
Utilities.Instrumentation.Common.LoggingContext loggingContext)
{
Contract.RequiresNotNull(repositories);
Contract.RequiresNotNull(discoverCredentialProvider);
m_cancellationToken = cancellationToken;
m_loggingContext = loggingContext;
m_repositories = repositories;
m_stringTable = stringTable;
m_discoverCredentialProvider = discoverCredentialProvider;
m_initializationResult = new Lazy<Task<Possible<bool>>>(async () => {
var logger = new StringBuilder();
try
{
return (await VSTSAuthenticationHelper.TryCreateSourceRepositories(m_repositories, m_discoverCredentialProvider, m_cancellationToken, logger))
.Then<bool>(sourceRepositories =>
{
var packageBaseAddressMutable = new Dictionary<Uri, PackageSourceCredential>();
foreach (var sourceRepository in sourceRepositories)
{
var serviceIndexResource = sourceRepository.GetResource<ServiceIndexResourceV3>(m_cancellationToken);
if (serviceIndexResource == null)
{
return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, $"Cannot find index service for ${sourceRepository.PackageSource.SourceUri}");
}
foreach (Uri packageBaseAddress in serviceIndexResource.GetServiceEntryUris(ServiceTypes.PackageBaseAddress))
{
packageBaseAddressMutable[packageBaseAddress] = sourceRepository.PackageSource.Credentials;
}
}
m_packageBaseAddress = packageBaseAddressMutable;
return true;
});
}
catch (Exception e) when (e is AggregateException || e is HttpRequestException)
{
return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository, e);
}
finally
{
var log = logger.ToString();
if (!string.IsNullOrWhiteSpace(log))
{
Logger.Log.NuGetInspectionInitializationInfo(m_loggingContext, log);
}
}
},
LazyThreadSafetyMode.ExecutionAndPublication);
}
/// <summary>
/// Whether <see cref="TryInitAsync"/> has been succesfully called
/// </summary>
/// <remarks>Thread safe</remarks>
public async Task<bool> IsInitializedAsync() => m_initializationResult.IsValueCreated && (await m_initializationResult.Value).Succeeded;
/// <summary>
/// Retrieves the index sources of the specified repositories and initializes the base addresses
/// </summary>
/// <remarks>Thread safe. Subsequents initializations have no effect and return the same result as the first one did.</remarks>
public Task<Possible<bool>> TryInitAsync() => m_initializationResult.Value;
/// <summary>
/// Tries to retrieve the layout and nuspec of the given package
/// </summary>
/// <remarks>Thread safe</remarks>
public async Task<Possible<NugetInspectedPackage>> TryInspectAsync(INugetPackage identity)
{
Contract.Assert(await IsInitializedAsync(), "TryInitAsync() must be succesfully called first");
if (m_packageBaseAddress.Count == 0)
{
return new NugetFailure(NugetFailure.FailureType.NoBaseAddressForRepository);
}
// Build the URI for the requested package and try to inspect it
var packageIdLowerCase = identity.Id.ToLowerInvariant();
var version = new NuGet.Versioning.NuGetVersion(identity.Version).ToNormalizedString();
Possible<NugetInspectedPackage> maybeInspectedPackage = default;
foreach (var baseAddress in m_packageBaseAddress)
{
// URIs for retrieving the nuspec and the nupkg
var packageUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.{version}.nupkg";
var nuspecUri = $"{baseAddress.Key.AbsoluteUri}{packageIdLowerCase}/{version}/{packageIdLowerCase}.nuspec";
maybeInspectedPackage = await TryInspectPackageAsync(identity, new Uri(packageUri), new Uri(nuspecUri), baseAddress.Value);
if (maybeInspectedPackage.Succeeded)
{
return maybeInspectedPackage;
}
}
return maybeInspectedPackage.Failure;
}
private async Task<Possible<NugetInspectedPackage>> TryInspectPackageAsync(INugetPackage identity, Uri nupkgUri, Uri nuspecUri, PackageSourceCredential packageSourceCredential)
{
AuthenticationHeaderValue authenticationHeader = null;
// If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
// We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
if (packageSourceCredential != null)
{
authenticationHeader = VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(packageSourceCredential.PasswordText);
}
// We want to be able to read the zip file central directory, where the layout of the package is. This is at the end of a zip file,
// so we'll start requesting partial chunks of the content starting from the end and increasingly request more until we can understand
// the zip central directory
int retries = 3;
while (true)
{
try
{
using (var httpClient = new HttpClient())
{
httpClient.Timeout = TimeSpan.FromMinutes(1);
// Use authentication if defined
if (authenticationHeader != null)
{
httpClient.DefaultRequestHeaders.Accept.Clear();
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = authenticationHeader;
}
// We need the nuspec file for analyzing the package
var response = await httpClient.GetAsync(nuspecUri, m_cancellationToken);
if (!response.IsSuccessStatusCode)
{
return NugetFailure.CreateNugetInvocationFailure(identity, response.ReasonPhrase);
}
var nuspec = await response.Content.ReadAsStringAsync();
// Now inspect the content of the nupkg
return (await InspectContentAsync(identity, nupkgUri, authenticationHeader, httpClient)).Then(content => new NugetInspectedPackage(nuspec, content));
}
}
catch (Exception e) when (e is HttpRequestException || e is AggregateException || e is TaskCanceledException)
{
retries--;
if (retries == 0)
{
return NugetFailure.CreateNugetInvocationFailure(identity, e);
}
}
}
}
private async Task<Possible<IReadOnlyList<RelativePath>>> InspectContentAsync(INugetPackage identity, Uri nupkgUri, AuthenticationHeaderValue authenticationHeader, HttpClient httpClient)
{
long? chunkStart = null;
HttpResponseMessage response;
bool forceFullDownload = false;
var partialPackage = new MemoryStream();
try
{
// How many chunks we downloaded so far for the given package
int chunkCount = 0;
while (true)
{
// Only set a download range if we are not forcing a full download
if (!forceFullDownload)
{
// Set the range header to retrieve a particular range of the content
if (!chunkStart.HasValue)
{
// We don't know the total size yet, this is the first request. Start with MinimalChunkSizeInBytes.
httpClient.DefaultRequestHeaders.Add("Range", "bytes=-" + MinimalChunkSizeInBytes);
}
else
{
// This is not the first time we request a chunk, and that means the content we retrieved son far is not enough to read the zip central directory.
// So we already know where the chunk starts (and ends)
httpClient.DefaultRequestHeaders.Add("Range", $"bytes={chunkStart}-{chunkStart + (chunkCount * IncrementChunkSizeInBytes) - 1}");
}
}
// TODO: redirect handling may be needed here
response = await httpClient.GetAsync(nupkgUri, HttpCompletionOption.ResponseHeadersRead, m_cancellationToken);
// In the rare case where the initial chunk is bigger than the package, we might get this. Just force full download and retry
if (response.StatusCode == System.Net.HttpStatusCode.RequestedRangeNotSatisfiable)
{
httpClient.DefaultRequestHeaders.Remove("Range");
forceFullDownload = true;
continue;
}
if (!response.IsSuccessStatusCode)
{
break;
}
// Check whether the service decided to ignore the partial request, and instead downloaded the whole thing
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
forceFullDownload = true;
}
long totalLength = 0;
if (!forceFullDownload)
{
totalLength = response.Content.Headers.ContentRange.Length.Value;
if (!chunkStart.HasValue)
{
// We just did the first request, chunk start points to MinimalChunkSizeInBytes from the end
chunkStart = totalLength - MinimalChunkSizeInBytes;
}
}
#if NET_COREAPP_60
using (var chunk = await response.Content.ReadAsStreamAsync(m_cancellationToken))
#else
using (var chunk = await response.Content.ReadAsStreamAsync())
#endif
{
// Unfortunately the .net framework does not support prepending a stream, so we do it manually
// TODO: If this becomes a perf/footprint issue we could write a stream wrapper that knows how to compose streams. But
// the expectation is that we shouldn't need to iterate over the same package for very long until we can read it
partialPackage = await PrependToStreamAsync(chunk, partialPackage);
}
// We don't want to get in the business of decoding a zip file. For that we use ZipArchive and check whether we get an exception
// when we read it.
// However, ZipArchive expects a stream with the proper length, since it does some validations about the location of the central
// directory baed on that. To avoid creating a stream of the real size (some packages are GB big), we use a ZeroPaddedStream that
// wraps the original stream but pretends to have the required size
IReadOnlyCollection<ZipArchiveEntry> entries;
try
{
var zf = new ZipArchive(forceFullDownload
? partialPackage
: new ZeroLeftPaddedStream(partialPackage, totalLength), ZipArchiveMode.Read);
entries = zf.Entries;
}
catch (InvalidDataException ex)
{
// We downloaded the package in full but we cannot recognize it as a zip
if (forceFullDownload)
{
return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {ex.ToStringDemystified()} ");
}
// This check is just a heuristics for the rare case when we already downloaded more than 10% of the package total
// size but we can't still read the zip central directory. At this point we can rather download the whole package
if (partialPackage.Length < totalLength * .1)
{
chunkCount++;
// We were not able to read the package central directory with what we downloaded so far. Request another chunk and
// try again
chunkStart -= chunkCount * IncrementChunkSizeInBytes;
httpClient.DefaultRequestHeaders.Remove("Range");
continue;
}
httpClient.DefaultRequestHeaders.Remove("Range");
forceFullDownload = true;
continue;
}
return entries
.Select(entry => entry.FullName.Contains('%') ? System.Net.WebUtility.UrlDecode(entry.FullName) : entry.FullName)
.Where(entry => PackageHelper.IsPackageFile(entry, PackageSaveMode.Files | PackageSaveMode.Nuspec))
.Select(entry => RelativePath.Create(m_stringTable, entry)).ToArray();
}
}
finally
{
partialPackage.Dispose();
}
return NugetFailure.CreateNugetInvocationFailure(identity, $"Cannot inspect package layout: {response.RequestMessage} ");
}
private async Task<MemoryStream> PrependToStreamAsync(Stream prefix, MemoryStream stream)
{
// If the destination stream is empty, just copy the prefix over
if (stream.Length == 0)
{
await prefix.CopyToAsync(stream, m_cancellationToken);
return stream;
}
var tempPackage = new MemoryStream();
await prefix.CopyToAsync(tempPackage, m_cancellationToken);
stream.Position = 0;
await stream.CopyToAsync(tempPackage, m_cancellationToken);
stream.Dispose();
return tempPackage;
}
}
}

102
Public/Src/FrontEnd/Nuget/NugetSpecGenerator.cs
Просмотреть файл

@ -14,6 +14,8 @@ using TypeScript.Net.Extensions;
using TypeScript.Net.Types;
using static TypeScript.Net.DScript.SyntaxFactory;
using static BuildXL.FrontEnd.Nuget.SyntaxFactoryEx;
using BuildXL.FrontEnd.Sdk;
using BuildXL.FrontEnd.Script.Literals;
namespace BuildXL.FrontEnd.Nuget
{
@ -25,22 +27,31 @@ namespace BuildXL.FrontEnd.Nuget
private readonly PathTable m_pathTable;
private readonly PackageOnDisk m_packageOnDisk;
private readonly NugetAnalyzedPackage m_analyzedPackage;
private readonly IReadOnlyDictionary<string, string> m_repositories;
private readonly NugetFrameworkMonikers m_nugetFrameworkMonikers;
private readonly AbsolutePath m_sourceDirectory;
private readonly PathAtom m_xmlExtension;
private readonly PathAtom m_pdbExtension;
private readonly int? m_timeoutInMinutes;
/// <summary>Current spec generation format version</summary>
public const int SpecGenerationFormatVersion = 11;
public const int SpecGenerationFormatVersion = 12;
/// <nodoc />
public NugetSpecGenerator(PathTable pathTable, NugetAnalyzedPackage analyzedPackage)
public NugetSpecGenerator(
PathTable pathTable,
NugetAnalyzedPackage analyzedPackage,
IReadOnlyDictionary<string, string> repositories,
AbsolutePath sourceDirectory,
int? timeoutInMinutes = null)
{
m_pathTable = pathTable;
m_analyzedPackage = analyzedPackage;
m_repositories = repositories;
m_packageOnDisk = analyzedPackage.PackageOnDisk;
m_nugetFrameworkMonikers = new NugetFrameworkMonikers(pathTable.StringTable);
m_sourceDirectory = sourceDirectory;
m_timeoutInMinutes = timeoutInMinutes;
m_xmlExtension = PathAtom.Create(pathTable.StringTable, ".xml");
m_pdbExtension = PathAtom.Create(pathTable.StringTable, ".pdb");
@ -53,13 +64,13 @@ namespace BuildXL.FrontEnd.Nuget
/// The generated format is:
/// [optional] import of managed sdk core
/// [optional] qualifier declaration
/// const packageRoot = d`absolute path to the package roo`;
/// @@public
/// export const contents: StaticDirectory = Transformer.sealDirectory(
/// packageRoot,
/// [
/// f`${packageRoot}/file`,
/// ]);
/// export const contents: StaticDirectory = NuGetDownloader.downloadPackage(
/// {
/// id: "package ID",
/// version: "X.XX",
/// ...
/// }
/// @@public
/// export const pkg: NugetPackage = {contents ...};
/// </remarks>
@ -68,8 +79,8 @@ namespace BuildXL.FrontEnd.Nuget
{
var sourceFileBuilder = new SourceFileBuilder();
// 0. Import {Transformer} from "Sdk.Transformers" to be able to seal directories
sourceFileBuilder.Statement(ImportDeclaration(new [] { "Transformer" }, "Sdk.Transformers"));
// 0. Import * as NugetDownloader from "BuildXL.Tools.NugetDownloader" to be able to download NuGet packages
sourceFileBuilder.Statement(ImportDeclaration("NugetDownloader", "BuildXL.Tools.NugetDownloader"));
// 1. Optional import of managed sdk.
if (analyzedPackage.IsManagedPackage)
@ -88,12 +99,7 @@ namespace BuildXL.FrontEnd.Nuget
.SemicolonAndBlankLine();
}
// 3. Declare a public directory that points to the package root, for convenience reasons
sourceFileBuilder
.Statement(new VariableDeclarationBuilder().Name("packageRoot").Initializer(PropertyAccess("Contents", "packageRoot")).Build())
.SemicolonAndBlankLine();
// Create a sealed directory declaration with all the package content
// Create a seal directory declaration with all the package content
sourceFileBuilder
.Statement(CreatePackageContents())
.SemicolonAndBlankLine();
@ -279,24 +285,59 @@ namespace BuildXL.FrontEnd.Nuget
private IStatement CreatePackageContents()
{
var relativepath = "../../../pkgs/" + m_packageOnDisk.Package.Id + "." + m_packageOnDisk.Package.Version;
// Arguments for calling the nuget downloader SDK
var downloadCallArgs = new List<(string, IExpression expression)>(4)
{
("id", new LiteralExpression(m_analyzedPackage.Id)),
("version", new LiteralExpression(m_analyzedPackage.Version)),
("downloadDirectory", Identifier("outputDir")),
("extractedFiles", new ArrayLiteralExpression(m_analyzedPackage.PackageOnDisk.Contents
.Select(relativePath => PathLikeLiteral(InterpolationKind.RelativePathInterpolation, relativePath.ToString(m_pathTable.StringTable, PathFormat.Script))))),
("repositories", new ArrayLiteralExpression(m_repositories.Select(kvp => new ArrayLiteralExpression(new LiteralExpression(kvp.Key), new LiteralExpression(kvp.Value)))))
};
// If a credential provider was used to inspect the package, pass it as an argument to be able to retrieve it.
if (m_analyzedPackage.CredentialProviderPath.IsValid)
{
// If the credential provider is within the source tree, express it in terms of a mount, so the generated
// spec is more resilient to cache hits across machines
IExpression path;
if (m_sourceDirectory.TryGetRelative(m_pathTable, m_analyzedPackage.CredentialProviderPath, out var relativeCredentialProviderPath))
{
path = PathLikeLiteral(
InterpolationKind.FileInterpolation,
new PropertyAccessExpression(new CallExpression(new PropertyAccessExpression("Context", "getMount"), new LiteralExpression("SourceRoot")), "path") ,
"/" + relativeCredentialProviderPath.ToString(m_pathTable.StringTable, PathFormat.Script));
}
else
{
path = PathLikeLiteral(InterpolationKind.FileInterpolation, m_analyzedPackage.CredentialProviderPath.ToString(m_pathTable, PathFormat.Script));
}
downloadCallArgs.Add(("credentialProviderPath", path));
}
if (m_timeoutInMinutes != null)
{
downloadCallArgs.Add(("timeoutInMinutes", new LiteralExpression(m_timeoutInMinutes.Value)));
}
return new ModuleDeclaration(
"Contents",
Qualifier(new TypeLiteralNode()),
PathLikeConstVariableDeclaration("packageRoot", InterpolationKind.DirectoryInterpolation, relativepath, Visibility.Export),
new VariableDeclarationBuilder().Name("outputDir").Visibility(Visibility.None).Type(new TypeReferenceNode("Directory")).Initializer(
new CallExpression(new PropertyAccessExpression("Context", "getNewOutputDirectory"), new LiteralExpression("nuget"))).Build(),
new VariableDeclarationBuilder()
.Name("all")
.Visibility(Visibility.Public)
.Type(new TypeReferenceNode("StaticDirectory"))
.Initializer(
new CallExpression(
new PropertyAccessExpression("Transformer", "sealDirectory"),
new Identifier("packageRoot"),
new ArrayLiteralExpression(
m_packageOnDisk.Contents.OrderBy(path => path.ToString(m_pathTable.StringTable)).Select(GetFileExpressionForPath)
.ToArray())))
new CallExpression(
new PropertyAccessExpression("NugetDownloader", "downloadPackage"),
ObjectLiteral(downloadCallArgs.ToArray())))
.Build()
);
}
@ -377,11 +418,10 @@ namespace BuildXL.FrontEnd.Nuget
private IExpression GetFileExpressionForPath(RelativePath relativePath)
{
// f`{packageRoot}/relativePath`
return PathLikeLiteral(
InterpolationKind.FileInterpolation,
Identifier("packageRoot"),
"/" + relativePath.ToString(m_pathTable.StringTable, PathFormat.Script));
// all.assertExistence(r`relativePath`)
return new CallExpression(new PropertyAccessExpression("Contents", "all", "getFile"), PathLikeLiteral(
InterpolationKind.RelativePathInterpolation,
relativePath.ToString(m_pathTable.StringTable, PathFormat.Script)));
}
private IExpression CreateSimpleBinary(RelativePath binaryFile)

127
Public/Src/FrontEnd/Nuget/Tracing/Log.cs
Просмотреть файл

@ -23,33 +23,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
/// </summary>
public static Logger Log { get; } = new LoggerImpl();
[GeneratedEvent(
(ushort)LogEventId.LaunchingNugetExe,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
Keywords = (int)Keywords.Diagnostics,
EventTask = (ushort)Tasks.Parser,
Message = "Package nuget://{id}/{version} is being restored by launching nuget.exe with commandline: {commandline}")]
public abstract void LaunchingNugetExe(LoggingContext context, string id, string version, string commandline);
[GeneratedEvent(
(ushort)LogEventId.CredentialProviderRequiresToolUrl,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError),
EventTask = (ushort)Tasks.Parser,
Message = "CredentialProviders for the Nuget resolver are required to specify ToolUrl. '{toolName}' does not do so.")]
public abstract void CredentialProviderRequiresToolUrl(LoggingContext context, string toolName);
[GeneratedEvent(
(ushort)LogEventId.NugetDownloadInvalidHash,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Warning,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message = "Configured downloadHash '{hash}' for tool '{toolName}' is invalid. Attempt to download tool without hash guard.")]
public abstract void NugetDownloadInvalidHash(LoggingContext context, string toolName, string hash);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedToCleanTargetFolder,
EventGenerators = EventGenerators.LocalOnly,
@ -60,74 +33,13 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
public abstract void NugetFailedToCleanTargetFolder(LoggingContext context, string id, string version, string targetLocation, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedWithNonZeroExitCode,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. \r\nTools output:\r\n{output}\r\nSee the buildxl log for more details.")]
public abstract void NugetFailedWithNonZeroExitCode(LoggingContext context, string id, string version, int exitcode, string output);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedDueToSomeWellKnownIssue,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be restored. {message}\r\nSee the buildxl log for more details.")]
public abstract void NugetFailedDueToSomeWellKnownIssue(LoggingContext context, string id, string version, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedWithNonZeroExitCodeDetailed,
(ushort)LogEventId.NugetInspectionInitialization,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be downloaded because nuget.exe failed with exit code '{exitcode}'. The standard output was:\r\n{stdOut}\r\n. The standard Error was:\r\n{stdErr}")]
public abstract void NugetFailedWithNonZeroExitCodeDetailed(
LoggingContext context,
string id,
string version,
int exitcode,
string stdOut,
string stdErr);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedToListPackageContents,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be downloaded because we could not list the content of the folder '{packageFolder}': {message}")]
public abstract void NugetFailedToListPackageContents(LoggingContext context, string id, string version, string packageFolder, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedToWriteConfigurationFile,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message = "Could not be write configuration file to '{configFile}': {message}")]
public abstract void NugetFailedToWriteConfigurationFile(LoggingContext context, string configFile, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedToWriteConfigurationFileForPackage,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be processed because we could not write configuration file to '{configFile}': {message}")]
public abstract void NugetFailedToWriteConfigurationFileForPackage(
LoggingContext context,
string id,
string version,
string configFile,
string message);
Message = "Nuget inspection info: {message}")]
public abstract void NuGetInspectionInitializationInfo(LoggingContext context, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedToWriteSpecFileForPackage,
@ -169,21 +81,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
"Package nuget://{id}/{version} could not be processed because the nuspec file cannot be found at '{expectedPath}'.")]
public abstract void NugetFailedNuSpecFileNotFound(LoggingContext context, string id, string version, string expectedPath);
[GeneratedEvent(
(ushort)LogEventId.NugetFailedWithInvalidNuSpecXml,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Error,
Keywords = (ushort)(Keywords.UserMessage | Keywords.UserError),
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be processed because the nuspec file at: '{nuspecFile}' has unexpected xml contents: {illegalXmlElement}")]
public abstract void NugetFailedWithInvalidNuSpecXml(
LoggingContext context,
string id,
string version,
string nuspecFile,
string illegalXmlElement);
[GeneratedEvent(
(ushort)LogEventId.NugetPackageVersionIsInvalid,
EventGenerators = EventGenerators.LocalOnly,
@ -193,16 +90,6 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
Message = "Invalid nuget version '{version}' found in config.dsc for package '{packageName}'. Expected version format is 'A.B.C.D'.")]
public abstract void ConfigNugetPackageVersionIsInvalid(LoggingContext context, string version, string packageName);
[GeneratedEvent(
(ushort)LogEventId.NugetLaunchFailed,
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
Keywords = (ushort)Keywords.UserMessage,
EventTask = (ushort)Tasks.Parser,
Message =
"Package nuget://{id}/{version} could not be downloaded because nuget.exe could not be launched: {message}")]
public abstract void NugetLaunchFailed(LoggingContext context, string id, string version, string message);
[GeneratedEvent(
(ushort)LogEventId.NugetStatistics,
EventGenerators = EventGenerators.LocalOnly,
@ -218,7 +105,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Informational,
EventTask = (ushort)Tasks.Parser,
Message = "Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).",
Message = "Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done).",
Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
public abstract void NugetPackageDownloadedCount(LoggingContext context, long packagesDownloadedCount, long totalPackagesCount);
@ -227,7 +114,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Informational,
EventTask = (ushort)Tasks.Parser,
Message = "Restored {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.",
Message = "Inspected {totalPackagesCount} NuGet packages in {totalMilliseconds}ms.",
Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
public abstract void NugetPackagesAreRestored(LoggingContext context, long totalPackagesCount, long totalMilliseconds);
@ -254,7 +141,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
EventGenerators = EventGenerators.LocalOnly,
EventLevel = Level.Verbose,
EventTask = (ushort)Tasks.Parser,
Message = "Package nuget://{id}/{version} could not be restored because of an unhandled error '{error}'.",
Message = "Package nuget://{id}/{version} could not be inspected because of an unhandled error '{error}'.",
Keywords = (int)Keywords.UserMessage | (int)Keywords.Overwritable)]
public abstract void NugetUnhandledError(LoggingContext context, string id, string version, string error);
@ -264,7 +151,7 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
EventLevel = Level.Informational,
EventTask = (ushort)Tasks.Parser,
Message =
"Restoring NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}",
"Inspecting NuGet packages ({packagesDownloadedCount} of {totalPackagesCount} done). Remaining {packagesToDownloadDetail}",
Keywords = (int)Keywords.UserMessage | (int)Keywords.OverwritableOnly)]
public abstract void NugetPackageDownloadedCountWithDetails(LoggingContext context, long packagesDownloadedCount,
long totalPackagesCount, string packagesToDownloadDetail);

71
Public/Src/FrontEnd/Nuget/Tracing/LogEventId.cs
Просмотреть файл

@ -14,40 +14,41 @@ namespace BuildXL.FrontEnd.Nuget.Tracing
None = 0,
// reserved 11300 .. 11400 for nuget
LaunchingNugetExe = 11300,
CredentialProviderRequiresToolUrl,
NugetDownloadInvalidHash,
NugetFailedToCleanTargetFolder,
NugetFailedWithNonZeroExitCode,
NugetFailedWithNonZeroExitCodeDetailed,
NugetFailedToListPackageContents,
NugetFailedToWriteConfigurationFile,
NugetFailedToWriteConfigurationFileForPackage,
NugetFailedToWriteSpecFileForPackage,
NugetFailedToReadNuSpecFile,
NugetFailedWithInvalidNuSpecXml,
NugetPackageVersionIsInvalid,
NugetLaunchFailed,
NugetStatistics,
NugetPackagesRestoredCount,
NugetPackagesAreRestored,
NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen,
NugetDependencyVersionWasPickedWithinRange,
NugetDependencyVersionDoesNotMatch,
NugetUnknownFramework,
NugetPackageDownloadDetails,
NugetFailedNuSpecFileNotFound,
NugetFailedDueToSomeWellKnownIssue,
NugetRegenerateNugetSpecs,
NugetRegeneratingNugetSpecs,
NugetUnhandledError,
ForcePopulateTheCacheOptionWasSpecified,
NugetConcurrencyLevel,
UsePackagesFromDisOptionWasSpecified,
NugetFailedDownloadPackagesAndGenerateSpecs,
NugetFailedDownloadPackage,
NugetFailedGenerationResultFromDownloadedPackage,
NugetFailedToWriteGeneratedSpecStateFile,
NugetCannotReuseSpecOnDisk,
// Reserved LaunchingNugetExe = 11300,
// Reserved CredentialProviderRequiresToolUrl = 11301,
// Reserved NugetDownloadInvalidHash = 11302,
NugetFailedToCleanTargetFolder = 11303,
// Reserved NugetFailedWithNonZeroExitCode = 11304,
// Reserved NugetFailedWithNonZeroExitCodeDetailed = 11305,
// Reserved NugetFailedToListPackageContents = 11306,
// Reserved NugetFailedToWriteConfigurationFile = 11307,
// Reserved NugetFailedToWriteConfigurationFileForPackage = 11308,
NugetFailedToWriteSpecFileForPackage = 11309,
NugetFailedToReadNuSpecFile = 11310,
// Reserved NugetFailedWithInvalidNuSpecXml = 11311,
NugetPackageVersionIsInvalid = 11312,
// Reserved NugetLaunchFailed = 11313,
NugetStatistics = 11314,
NugetPackagesRestoredCount = 11315,
NugetPackagesAreRestored = 11316,
NugetDependencyVersionWasNotSpecifiedButConfigOneWasChosen = 11317,
NugetDependencyVersionWasPickedWithinRange = 11318,
NugetDependencyVersionDoesNotMatch = 11319,
NugetUnknownFramework = 11320,
NugetPackageDownloadDetails = 11321,
NugetFailedNuSpecFileNotFound = 11322,
// Reserved NugetFailedDueToSomeWellKnownIssue = 11323,
NugetRegenerateNugetSpecs = 11324,
NugetRegeneratingNugetSpecs = 11325,
NugetUnhandledError = 11326,
ForcePopulateTheCacheOptionWasSpecified = 11327,
NugetConcurrencyLevel = 11328,
UsePackagesFromDisOptionWasSpecified = 11329,
NugetFailedDownloadPackagesAndGenerateSpecs = 11330,
NugetFailedDownloadPackage = 11331,
NugetFailedGenerationResultFromDownloadedPackage = 11332,
NugetFailedToWriteGeneratedSpecStateFile = 11333,
NugetCannotReuseSpecOnDisk = 11334,
NugetInspectionInitialization = 11335,
}
}

805
Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs
Просмотреть файл

@ -62,6 +62,8 @@ namespace BuildXL.FrontEnd.Nuget
private const string SpecGenerationVersionFileSuffix = ".version";
private const string NugetCredentialProviderEnv = "NUGET_CREDENTIALPROVIDERS_PATH";
private NugetFrameworkMonikers m_nugetFrameworkMonikers;
// These are set during Initialize
@ -82,8 +84,6 @@ namespace BuildXL.FrontEnd.Nuget
private NugetResolverOutputLayout m_resolverOutputLayout;
private IConfiguration m_configuration;
private readonly Lazy<AbsolutePath> m_nugetToolFolder;
/// <inheritdoc />
public string Kind => KnownResolverKind.NugetResolverKind;
@ -101,10 +101,6 @@ namespace BuildXL.FrontEnd.Nuget
m_embeddedSpecsResolver = new WorkspaceSourceModuleResolver(stringTable, statistics, logger: null);
m_useMonoBasedNuGet = OperatingSystemHelper.IsUnixOS;
m_nugetToolFolder = new Lazy<AbsolutePath>(
() => m_host.GetFolderForFrontEnd(NugetResolverName).Combine(PathTable, "nuget"),
LazyThreadSafetyMode.ExecutionAndPublication);
}
/// <inheritdoc/>
@ -393,21 +389,58 @@ namespace BuildXL.FrontEnd.Nuget
private Task<Possible<NugetGenerationResult>> DownloadPackagesAndGenerateSpecsIfNeededInternal()
{
return m_nugetGenerationResult.GetOrCreate(this, @this => @this.DownloadPackagesAndGenerateSpecsAsync());
return m_nugetGenerationResult.GetOrCreate(this, @this => Task.FromResult(@this.DownloadPackagesAndGenerateSpecs()));
}
private AbsolutePath GetNugetToolFolder()
private Possible<AbsolutePath> TryResolveCredentialProvider()
{
return m_nugetToolFolder.Value;
if (!m_host.Engine.TryGetBuildParameter(NugetCredentialProviderEnv, nameof(NugetFrontEnd), out string credentialProvidersPaths))
{
return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider, $"Environment variable {NugetCredentialProviderEnv} is not set");
}
// Here we do something slightly simpler than what NuGet does and just look for the first credential
// provider we can find
AbsolutePath credentialProviderPath = AbsolutePath.Invalid;
foreach (string path in credentialProvidersPaths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
if (!AbsolutePath.TryCreate(m_context.PathTable, path, out var absolutePath))
{
break;
}
// Use the engine to enumerate, since the result of the enumeration should be sensitive
// to the graph building process
credentialProviderPath = m_host.Engine.EnumerateFiles(absolutePath, "CredentialProvider*.exe").FirstOrDefault();
if (credentialProviderPath.IsValid)
{
break;
}
}
if (!credentialProviderPath.IsValid)
{
return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider, $"Unable to authenticate using a credential provider: Credential provider was not found under '{credentialProvidersPaths}'.");
}
// We want to rebuild the build graph if the credential provider changed. Running the whole auth process under detours sounds like too much,
// let's just read the credential provider main .exe so presence/hash gets recorded instead.
m_host.Engine.TryGetFrontEndFile(credentialProviderPath, nameof(NugetFrontEnd), out _);
return m_host.Engine.Translate(credentialProviderPath);
}
private AbsolutePath GetNugetConfigPath()
private Possible<NugetGenerationResult> DownloadPackagesAndGenerateSpecs()
{
return GetNugetToolFolder().Combine(PathTable, "NuGet.config");
}
var maybeCredentialProviderPath = TryResolveCredentialProvider();
var nugetInspector = new NugetPackageInspector(
m_resolverSettings.Repositories.Select(kvp => (kvp.Key, new Uri(kvp.Value))),
PathTable.StringTable,
() => maybeCredentialProviderPath.Then(path => path.ToString(PathTable)),
m_context.CancellationToken,
m_context.LoggingContext);
private async Task<Possible<NugetGenerationResult>> DownloadPackagesAndGenerateSpecsAsync()
{
// Log if the full package restore is requested.
if (m_configuration.FrontEnd.ForcePopulatePackageCache())
{
@ -429,26 +462,6 @@ namespace BuildXL.FrontEnd.Nuget
using (var nugetEndToEndStopWatch = m_statistics.EndToEnd.Start())
{
var possiblePaths = await TryDownloadNugetAsync(m_resolverSettings.Configuration, GetNugetToolFolder());
if (!possiblePaths.Succeeded)
{
Logger.Log.NugetFailedDownloadPackagesAndGenerateSpecs(m_context.LoggingContext, possiblePaths.Failure.DescribeIncludingInnerFailures());
return possiblePaths.Failure;
}
var possibleNugetConfig = CreateNuGetConfig(m_repositories);
var possibleNuGetConfig = TryWriteXmlConfigFile(
package: null,
targetFile: GetNugetConfigPath(),
xmlDoc: possibleNugetConfig.Result);
if (!possibleNuGetConfig.Succeeded)
{
Logger.Log.NugetFailedDownloadPackagesAndGenerateSpecs(m_context.LoggingContext, possibleNuGetConfig.Failure.DescribeIncludingInnerFailures());
return possibleNuGetConfig.Failure;
}
// Will contain all packages successfully downloaded and analyzed
var restoredPackagesById = new Dictionary<string, NugetAnalyzedPackage>();
@ -468,6 +481,11 @@ namespace BuildXL.FrontEnd.Nuget
{
var aggregateResult = new Possible<NugetAnalyzedPackage>[nugetProgress.Length];
if (!m_host.Engine.TryGetBuildParameter(NugetCredentialProviderEnv, nameof(NugetFrontEnd), out string allCredentialProviderPaths))
{
allCredentialProviderPaths = string.Empty;
}
Parallel.For(fromInclusive: 0,
toExclusive: aggregateResult.Length,
new ParallelOptions()
@ -475,9 +493,20 @@ namespace BuildXL.FrontEnd.Nuget
MaxDegreeOfParallelism = concurrencyLevel,
CancellationToken = m_context.CancellationToken,
},
(index) =>
(index, state) =>
{
aggregateResult[index] = TryRestorePackageAsync(nugetProgress[index], possiblePaths.Result).GetAwaiter().GetResult();
aggregateResult[index] = TryInspectPackageAsync(
nugetProgress[index],
maybeCredentialProviderPath.Succeeded ? maybeCredentialProviderPath.Result : AbsolutePath.Invalid,
allCredentialProviderPaths,
nugetInspector).GetAwaiter().GetResult();
// Let's not schedule more work in the parallel for if one of the inspections failed
if (!aggregateResult[index].Succeeded)
{
state.Stop();
}
return;
});
@ -549,12 +578,12 @@ namespace BuildXL.FrontEnd.Nuget
{
if (m_configuration.DoesSourceDiskDriveHaveSeekPenalty(PathTable))
{
nugetConcurrency = Environment.ProcessorCount / 2;
nugetConcurrency = Environment.ProcessorCount;
message = I($"Lowering restore package concurrency to {nugetConcurrency} because a source drive is on HDD.");
}
else
{
nugetConcurrency = Math.Min(16, Environment.ProcessorCount * 2);
nugetConcurrency = Math.Min(128, Environment.ProcessorCount * 4);
message = I($"Increasing restore package concurrency to {nugetConcurrency} because a source drive is on SSD.");
}
}
@ -564,9 +593,11 @@ namespace BuildXL.FrontEnd.Nuget
return nugetConcurrency;
}
private async Task<Possible<NugetAnalyzedPackage>> TryRestorePackageAsync(
private async Task<Possible<NugetAnalyzedPackage>> TryInspectPackageAsync(
NugetProgress progress,
IReadOnlyList<AbsolutePath> credentialProviderPaths)
AbsolutePath selectedCredentialProviderPath,
string allCredentialProviderPaths,
NugetPackageInspector nugetInspector)
{
progress.StartRunning();
@ -579,11 +610,9 @@ namespace BuildXL.FrontEnd.Nuget
var layout = NugetPackageOutputLayout.Create(
PathTable,
package,
nugetTool: credentialProviderPaths[0],
nugetConfig: GetNugetConfigPath(),
resolverLayout: m_resolverOutputLayout);
var possiblePkg = await TryRestorePackageWithCache(package, progress, layout, credentialProviderPaths);
var possiblePkg = await TryInpectPackageWithCache(package, progress, layout, allCredentialProviderPaths, nugetInspector);
if (!possiblePkg.Succeeded)
{
@ -596,6 +625,7 @@ namespace BuildXL.FrontEnd.Nuget
var analyzedPackage = AnalyzeNugetPackage(
possiblePkg.Result,
selectedCredentialProviderPath,
m_resolverSettings.DoNotEnforceDependencyVersions);
if (!analyzedPackage.Succeeded)
{
@ -780,60 +810,6 @@ namespace BuildXL.FrontEnd.Nuget
return true;
}
private Possible<XDocument, NugetFailure> CreateNuGetConfig(IReadOnlyDictionary<string, string> repositories)
{
XElement credentials = null;
if (m_useMonoBasedNuGet)
{
var localNuGetConfigPath = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
".config",
"NuGet",
"NuGet.Config");
try
{
// Sadly nuget goes all over the disk to chain configs, but when it comes to the credentials it decides not to properly merge them.
// So for now we have to hack and read the credentials from the users profile and stick them in the local config....
if (FileUtilities.Exists(localNuGetConfigPath))
{
ExceptionUtilities.HandleRecoverableIOException(
() =>
{
var doc = XDocument.Load(localNuGetConfigPath);
credentials = doc.Element("configuration")?.Element("packageSourceCredentials");
},
e => throw new BuildXLException($"Failed to load nuget config {localNuGetConfigPath}", e));
}
}
catch (BuildXLException e)
{
Logger.Log.NugetFailedToWriteConfigurationFile(
m_context.LoggingContext,
localNuGetConfigPath,
e.LogEventMessage);
return new NugetFailure(NugetFailure.FailureType.WriteConfigFile, e.InnerException);
}
}
return new XDocument(
new XElement(
"configuration",
new XElement(
"packageRestore",
new XElement("clear"),
new XElement("add", new XAttribute("key", "enabled"), new XAttribute("value", "True"))),
new XElement(
"disabledPackageSources",
new XElement("clear")),
credentials,
new XElement(
"packageSources",
new XElement("clear"),
repositories.Select(
kv => new XElement("add", new XAttribute("key", kv.Key), new XAttribute("value", kv.Value))))));
}
private Possible<AbsolutePath> TryWriteSourceFile(INugetPackage package, AbsolutePath targetFile, ISourceFile sourceFile)
{
Contract.Requires(package != null);
@ -873,45 +849,6 @@ namespace BuildXL.FrontEnd.Nuget
return targetFile;
}
private Possible<AbsolutePath> TryWriteXmlConfigFile(INugetPackage package, AbsolutePath targetFile, XDocument xmlDoc)
{
var targetFilePath = targetFile.ToString(PathTable);
try
{
FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
ExceptionUtilities.HandleRecoverableIOException(
() =>
xmlDoc.Save(targetFilePath, SaveOptions.DisableFormatting),
e =>
{
throw new BuildXLException("Cannot save document", e);
});
}
catch (BuildXLException e)
{
if (package == null)
{
Logger.Log.NugetFailedToWriteConfigurationFile(m_context.LoggingContext, targetFilePath, e.LogEventMessage);
}
else
{
Logger.Log.NugetFailedToWriteConfigurationFileForPackage(
m_context.LoggingContext,
package.Id,
package.Version,
targetFilePath,
e.LogEventMessage);
}
return new NugetFailure(package, NugetFailure.FailureType.WriteConfigFile, e.InnerException);
}
m_host.Engine.RecordFrontEndFile(targetFile, NugetResolverName);
return targetFile;
}
private AbsolutePath GetPackageSpecDir(NugetAnalyzedPackage analyzedPackage)
{
return m_resolverOutputLayout.GeneratedSpecsFolder
@ -1008,8 +945,9 @@ namespace BuildXL.FrontEnd.Nuget
// No-op if the directory exists
FileUtilities.CreateDirectory(packageSpecDirStr);
var nugetSpecGenerator = new NugetSpecGenerator(PathTable, analyzedPackage);
var nugetSpecGenerator = new NugetSpecGenerator(PathTable, analyzedPackage, m_resolverSettings.Repositories,
m_configuration.Layout.SourceDirectory, m_resolverSettings.Configuration.DownloadTimeoutMin);
var possibleProjectFile = TryWriteSourceFile(
analyzedPackage.PackageOnDisk.Package,
@ -1044,6 +982,7 @@ namespace BuildXL.FrontEnd.Nuget
internal Possible<NugetAnalyzedPackage> AnalyzeNugetPackage(
PackageOnDisk packageOnDisk,
AbsolutePath credentialProviderPath,
bool doNotEnforceDependencyVersions)
{
Contract.Requires(packageOnDisk != null);
@ -1057,7 +996,7 @@ namespace BuildXL.FrontEnd.Nuget
}
var result = NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_nugetFrameworkMonikers, maybeNuspecXdoc.Result,
packageOnDisk, m_packageRegistry.AllPackagesById, doNotEnforceDependencyVersions);
packageOnDisk, m_packageRegistry.AllPackagesById, doNotEnforceDependencyVersions, credentialProviderPath);
if (result == null)
{
@ -1119,8 +1058,7 @@ namespace BuildXL.FrontEnd.Nuget
}
}
private string CreateRestoreFingerPrint(INugetPackage package, IEnumerable<AbsolutePath> credentialProviderPaths)
private string CreateRestoreFingerPrint(INugetPackage package, string credentialProviderPaths)
{
var fingerprintParams = new List<string>
{
@ -1130,7 +1068,7 @@ namespace BuildXL.FrontEnd.Nuget
};
if (credentialProviderPaths != null)
{
fingerprintParams.Add("cred=" + UppercaseSortAndJoinStrings(credentialProviderPaths.Select(p => p.ToString(PathTable))));
fingerprintParams.Add("cred=" + credentialProviderPaths);
}
return "nuget://" + string.Join("&", fingerprintParams);
@ -1149,11 +1087,12 @@ namespace BuildXL.FrontEnd.Nuget
return restoreFingerPrint + "&" + string.Join("&", fingerprintParams);
}
private async Task<Possible<PackageOnDisk>> TryRestorePackageWithCache(
private async Task<Possible<PackageOnDisk>> TryInpectPackageWithCache(
INugetPackage package,
NugetProgress progress,
NugetPackageOutputLayout layout,
IEnumerable<AbsolutePath> credentialProviderPaths)
string credentialProviderPaths,
NugetPackageInspector nugetInspector)
{
var packageRestoreFingerprint = CreateRestoreFingerPrint(package, credentialProviderPaths);
var identity = PackageIdentity.Nuget(package.Id, package.Version, package.Alias);
@ -1165,10 +1104,23 @@ namespace BuildXL.FrontEnd.Nuget
packageRestoreFingerprint,
identity,
layout.PackageFolder,
() =>
layout.PathToNuspec,
async () =>
{
progress.StartDownloadFromNuget();
return TryDownloadPackage(package, layout, credentialProviderPaths);
// We want to delay initialization until the first inspection that is actually needed
// Initializing the inspector involves resolving the index service for each specified repository
if (!await nugetInspector.IsInitializedAsync())
{
var initResult = await nugetInspector.TryInitAsync();
if (!initResult.Succeeded)
{
return initResult.Failure;
}
}
return await TryInspectPackage(package, layout, nugetInspector);
});
return maybePackage.Then(downloadResult =>
@ -1183,34 +1135,71 @@ namespace BuildXL.FrontEnd.Nuget
return string.Join(",", values.Select(s => s.ToUpperInvariant()).OrderBy(s => s));
}
private async Task<Possible<IReadOnlyList<RelativePath>>> TryDownloadPackage(
INugetPackage package, NugetPackageOutputLayout layout, IEnumerable<AbsolutePath> credentialProviderPaths)
private async Task<Possible<IReadOnlyList<RelativePath>>> TryInspectPackage(
INugetPackage package,
NugetPackageOutputLayout layout,
NugetPackageInspector nugetInspector)
{
var xmlConfigResult = TryWriteXmlConfigFile(package, layout.PackagesConfigFile, GetPackagesXml(package));
if (!xmlConfigResult.Succeeded)
{
return xmlConfigResult.Failure;
}
var cleanUpResult = TryCleanupPackagesFolder(package, layout);
if (!cleanUpResult.Succeeded)
{
return cleanUpResult.Failure;
}
var nugetExeResult = await TryLaunchNugetExeAsync(package, layout, credentialProviderPaths);
if (!nugetExeResult.Succeeded)
// Inspect the package (get nuspec and layout)
var maybeInspectedPackage = await nugetInspector.TryInspectAsync(package);
if (!maybeInspectedPackage.Succeeded)
{
return nugetExeResult.Failure;
return maybeInspectedPackage.Failure;
}
var contentResult = TryEnumerateDirectory(package, layout.PackageDirectory);
if (!contentResult.Succeeded)
{
return contentResult.Failure;
}
var inspectedPackage = maybeInspectedPackage.Result;
return contentResult.Result;
// Serialize the nuspec to disk. In this way we can also use the bxl cache to avoid
// downloading this content again. The hash file will contain the layout, which is serialized
// later
try
{
#if NET_FRAMEWORK
return ExceptionUtilities.HandleRecoverableIOException(
() =>
#else
return await ExceptionUtilities.HandleRecoverableIOException(
async () =>
#endif
{
FileUtilities.CreateDirectoryWithRetry(layout.PackageFolder.ToString(PathTable));
// XML files need to be serialized with the right enconding, so let's use XDocument
// for that
var xdocument = XDocument.Parse(inspectedPackage.Nuspec);
using (var nuspec = new FileStream(layout.PathToNuspec.ToString(PathTable), FileMode.Create))
{
#if NET_FRAMEWORK
xdocument.Save(nuspec);
#else
await xdocument.SaveAsync(nuspec, SaveOptions.None, m_context.CancellationToken);
#endif
}
return new Possible<IReadOnlyList<RelativePath>>(inspectedPackage.Content);
},
e =>
{
throw new BuildXLException("Cannot write package's nuspec file to disk", e);
});
}
catch (BuildXLException e)
{
Logger.Log.NugetFailedToWriteSpecFileForPackage(
m_context.LoggingContext,
package.Id,
package.Version,
layout.PathToNuspec.ToString(PathTable),
e.LogEventMessage);
return new NugetFailure(package, NugetFailure.FailureType.WriteSpecFile, e.InnerException);
}
}
private Possible<Unit> TryCleanupPackagesFolder(INugetPackage package, NugetPackageOutputLayout layout)
@ -1255,466 +1244,6 @@ namespace BuildXL.FrontEnd.Nuget
}
}
private async Task<Possible<Unit>> TryLaunchNugetExeAsync(INugetPackage package, NugetPackageOutputLayout layout, IEnumerable<AbsolutePath> credentialProviderPaths)
{
var fileAccessManifest = GenerateFileAccessManifest(layout, credentialProviderPaths);
var buildParameters = BuildParameters
.GetFactory()
.PopulateFromEnvironment();
var tool = layout.NugetTool.ToString(PathTable);
var argumentsBuilder = new StringBuilder();
if (m_useMonoBasedNuGet)
{
argumentsBuilder.AppendFormat("\"{0}\"", tool);
argumentsBuilder.Append(" ");
if (!buildParameters.ToDictionary().TryGetValue("MONO_HOME", out var monoHome))
{
return new NugetFailure(package, NugetFailure.FailureType.MissingMonoHome);
}
tool = Path.Combine(monoHome, "mono");
}
// TODO:escape quotes properly
argumentsBuilder
.AppendFormat("restore \"{0}\"", layout.PackagesConfigFile.ToString(PathTable))
.AppendFormat(" -OutputDirectory \"{0}\"", layout.PackageRootFolder.ToString(PathTable))
.Append(" -Verbosity detailed")
.AppendFormat(" -ConfigFile \"{0}\"", layout.NugetConfig.ToString(PathTable))
.Append(" -PackageSaveMode nuspec")
.Append(" -NoCache")
// Currently we have to hack nuget to MsBuild version 4 which should come form the current CLR.
.Append(" -MsBuildVersion 4");
if (!m_host.Configuration.Interactive)
{
// Prevent Nuget from showing any UI when not in interactive mode
argumentsBuilder.Append(" -NonInteractive");
}
var arguments = argumentsBuilder.ToString();
Logger.Log.LaunchingNugetExe(m_context.LoggingContext, package.Id, package.Version, tool + " " + arguments);
try
{
// For NugetFrontEnd always create a new ConHost process.
// The NugetFrontEnd is normally executed only once, so the overhead is low.
// Also the NugetFrontEnd is a really long running process, so creating the ConHost is relatively
// very cheap. It provides guarantee if the process pollutes the ConHost env,
// it will not affect the server ConHost.
var info =
new SandboxedProcessInfo(
m_context.PathTable,
new NugetFileStorage(layout.PackageTmpDirectory),
tool,
fileAccessManifest,
disableConHostSharing: true,
ContainerConfiguration.DisabledIsolation,
loggingContext: m_context.LoggingContext,
sandboxConnection: m_useMonoBasedNuGet ? new SandboxConnectionFake() : null)
{
Arguments = arguments,
WorkingDirectory = layout.TempDirectory.ToString(PathTable),
PipSemiStableHash = 0,
PipDescription = "NuGet FrontEnd",
EnvironmentVariables = GetNugetEnvironmentVariables(),
Timeout = TimeSpan.FromMinutes(20), // Limit the time nuget has to download each nuget package
};
return await RetryOnFailure(
runNuget: async () =>
{
var process = await SandboxedProcessFactory.StartAsync(info, forceSandboxing: !m_useMonoBasedNuGet);
var result = await process.GetResultAsync();
return (result, result.ExitCode == 0);
},
onError: async result =>
{
// Log the result before trying again
var (stdOut, stdErr) = await GetStandardOutAndError(result);
Logger.Log.NugetFailedWithNonZeroExitCodeDetailed(
m_context.LoggingContext,
package.Id,
package.Version,
result.ExitCode,
stdOut,
stdErr);
return (stdOut, stdErr);
},
onFinalFailure: (exitCode, stdOut, stdErr) =>
{
// Give up and fail
return NugetFailure.CreateNugetInvocationFailure(package, exitCode, stdOut, stdErr);
});
}
catch (BuildXLException e)
{
Logger.Log.NugetLaunchFailed(m_context.LoggingContext, package.Id, package.Version, e.LogEventMessage);
return new NugetFailure(package, NugetFailure.FailureType.NugetFailedWithIoException);
}
catch (Exception e)
{
return new NugetFailure(package, NugetFailure.FailureType.NugetFailedWithIoException, e);
}
async Task<(string stdOut, string stdErr)> GetStandardOutAndError(SandboxedProcessResult result)
{
try
{
await result.StandardOutput.SaveAsync();
var stdOut = await result.StandardOutput.ReadValueAsync();
await result.StandardError.SaveAsync();
var stdErr = await result.StandardError.ReadValueAsync();
return (stdOut, stdErr);
}
catch (BuildXLException e)
{
return (e.LogEventMessage, string.Empty);
}
}
BuildParameters.IBuildParameters GetNugetEnvironmentVariables()
{
// the environment variable names below should use the casing appropriate for the target OS
// (on Windows it won't matter, but on Unix-like systems, including Cygwin environment on Windows,
// it matters, and has to be all upper-cased). See also doc comment for IBuildParameters.Select
return buildParameters
.Select(
new[]
{
"ComSpec",
"PATH",
"PATHEXT",
"NUMBER_OF_PROCESSORS",
"OS",
"PROCESSOR_ARCHITECTURE",
"PROCESSOR_IDENTIFIER",
"PROCESSOR_LEVEL",
"PROCESSOR_REVISION",
"SystemDrive",
"SystemRoot",
"SYSTEMTYPE",
"NUGET_CREDENTIALPROVIDERS_PATH",
"__CLOUDBUILD_AUTH_HELPER_CONFIG__",
"__Q_DPAPI_Secrets_Dir",
// Nuget Credential Provider env variables
"1ESSHAREDASSETS_BUILDXL_FEED_PAT",
"CLOUDBUILD_BUILDXL_SELFHOST_FEED_PAT",
// Auth material needed for low-privilege build.
"QAUTHMATERIALROOT",
// Used by the artifacts credential provider. See here for more information on how this variable is configured - https://github.com/microsoft/artifacts-credprovider#azure-devops-server
"VSS_NUGET_EXTERNAL_FEED_ENDPOINTS"
})
.Override(
new Dictionary<string, string>()
{
{"TMP", layout.TempDirectoryAsString},
{"TEMP", layout.TempDirectoryAsString},
{"NUGET_PACKAGES", layout.TempDirectoryAsString},
{"NUGET_ROOT", layout.TempDirectoryAsString},
});
}
}
private class SandboxConnectionFake : ISandboxConnection
{
public SandboxKind Kind => SandboxKind.MacOsKext;
public int NumberOfKextConnections => 1;
public ulong MinReportQueueEnqueueTime { get; set; }
public TimeSpan CurrentDrought
{
get
{
var nowNs = Sandbox.GetMachAbsoluteTime();
var minReportTimeNs = MinReportQueueEnqueueTime;
return TimeSpan.FromTicks(nowNs > minReportTimeNs ? (long)((nowNs - minReportTimeNs) / 100) : 0);
}
}
public void Dispose() { }
public bool IsInTestMode => true;
public bool NotifyUsage(uint cpuUsage, uint availableRamMB) { return true; }
public bool NotifyPipStarted(LoggingContext loggingContext, FileAccessManifest fam, SandboxedProcessUnix process) { return true; }
public IEnumerable<(string, string)> AdditionalEnvVarsToSet(long pipId)
{
return Enumerable.Empty<(string, string)>();
}
public void NotifyPipProcessTerminated(long pipId, int processId) { }
public void NotifyRootProcessExited(long pipId, SandboxedProcessUnix process) { }
public bool NotifyPipFinished(long pipId, SandboxedProcessUnix process) { return true; }
public void ReleaseResources() { }
}
private static async Task<Possible<Unit>> RetryOnFailure(
Func<Task<(SandboxedProcessResult result, bool isPassed)>> runNuget,
Func<SandboxedProcessResult, Task<(string, string)>> onError,
Func<int, string, string, Possible<Unit>> onFinalFailure,
int retryCount = MaxRetryCount,
int retryDelayMs = RetryDelayMs)
{
Contract.Assert(retryCount >= 1, "Maximum retry count must be greater than or equal to one. Found " + retryCount);
var pass = false;
var iteration = 1;
while (!pass)
{
var tuple = await runNuget();
var result = tuple.result;
pass = tuple.isPassed;
if (!pass)
{
var (stdOut, stdErr) = await onError(result);
if (iteration >= retryCount)
{
return onFinalFailure(result.ExitCode, stdOut, stdErr);
}
// Try again!
iteration++;
await Task.Delay(retryDelayMs);
}
}
return Unit.Void;
}
private FileAccessManifest GenerateFileAccessManifest(NugetPackageOutputLayout layout, IEnumerable<AbsolutePath> credentialProviderPaths)
{
var fileAccessManifest = new FileAccessManifest(PathTable)
{
// TODO: If this is set to true, then NuGet will fail if TMG Forefront client is running.
// Filtering out in SandboxedProcessReport won't work because Detours already blocks the access to FwcWsp.dll.
// Almost all machines in Office run TMG Forefront client.
// So far for WDG, FailUnexpectedFileAccesses is false due to allowlists.
// As a consequence, the file access manifest below gets nullified.
FailUnexpectedFileAccesses = false,
ReportFileAccesses = true,
MonitorNtCreateFile = true,
MonitorZwCreateOpenQueryFile = true,
};
fileAccessManifest.AddScope(layout.TempDirectory, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
fileAccessManifest.AddScope(layout.PackageRootFolder, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
if (!OperatingSystemHelper.IsUnixOS)
{
fileAccessManifest.AddScope(
AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.Windows)),
FileAccessPolicy.MaskAll,
FileAccessPolicy.AllowAllButSymlinkCreation);
}
fileAccessManifest.AddPath(layout.NugetTool, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing);
fileAccessManifest.AddPath(layout.NugetToolExeConfig, values: FileAccessPolicy.AllowReadIfNonexistent, mask: FileAccessPolicy.MaskNothing);
fileAccessManifest.AddPath(layout.NugetConfig, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing);
fileAccessManifest.AddPath(layout.PackagesConfigFile, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing);
// Nuget is picky
fileAccessManifest.AddScope(layout.ResolverFolder, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
// Nuget fails if it can't access files in the users (https://github.com/NuGet/Home/issues/2676) profile.
// We'll have to explicitly set all config in our nuget.config file and override anything the user can set and allow the read here.
var roamingAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.ApplicationData)).Combine(PathTable, "NuGet");
fileAccessManifest.AddScope(roamingAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
var localAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.LocalApplicationData)).Combine(PathTable, "NuGet");
fileAccessManifest.AddScope(localAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
// Nuget also probes in ProgramData on the machine.
var commonAppDataNuget = AbsolutePath.Create(PathTable, SpecialFolderUtilities.GetFolderPath(Environment.SpecialFolder.CommonApplicationData)).Combine(PathTable, "NuGet");
fileAccessManifest.AddScope(commonAppDataNuget, FileAccessPolicy.MaskAll, FileAccessPolicy.AllowAllButSymlinkCreation);
foreach (var providerPath in credentialProviderPaths)
{
fileAccessManifest.AddPath(providerPath, values: FileAccessPolicy.AllowRead, mask: FileAccessPolicy.MaskNothing);
fileAccessManifest.AddPath(
providerPath.ChangeExtension(PathTable, PathAtom.Create(PathTable.StringTable, ".exe.config")),
values: FileAccessPolicy.AllowRead,
mask: FileAccessPolicy.MaskNothing);
}
return fileAccessManifest;
}
private static XDocument GetPackagesXml(INugetPackage package)
{
var version = package.Version;
var plusIndex = version.IndexOf('+');
if (plusIndex > 0)
{
version = version.Substring(0, plusIndex);
}
return new XDocument(
new XElement(
"packages",
new XElement(
"package",
new XAttribute("id", package.Id),
new XAttribute("version", version))));
}
private Possible<List<RelativePath>, NugetFailure> TryEnumerateDirectory(INugetPackage package, string packagePath)
{
var enumerateDirectoryResult = EnumerateDirectoryRecursively(packagePath, out var contents);
if (!enumerateDirectoryResult.Succeeded)
{
var message = enumerateDirectoryResult.GetNativeErrorMessage();
Logger.Log.NugetFailedToListPackageContents(m_context.LoggingContext, package.Id, package.Version, packagePath, message);
return new NugetFailure(package, NugetFailure.FailureType.ListPackageContents, enumerateDirectoryResult.CreateExceptionForError());
}
return contents;
}
private EnumerateDirectoryResult EnumerateDirectoryRecursively(string packagePath, out List<RelativePath> resultingContent)
{
resultingContent = new List<RelativePath>();
return EnumerateDirectoryRecursively(RelativePath.Empty, resultingContent);
EnumerateDirectoryResult EnumerateDirectoryRecursively(RelativePath relativePath, List<RelativePath> contents)
{
var result = FileUtilities.EnumerateDirectoryEntries(
Path.Combine(packagePath, relativePath.ToString(m_context.StringTable)),
(name, attr) =>
{
var nestedRelativePath = relativePath.Combine(PathAtom.Create(m_context.StringTable, name));
if ((attr & FileAttributes.Directory) != 0)
{
EnumerateDirectoryRecursively(nestedRelativePath, contents);
}
else
{
contents.Add(nestedRelativePath);
}
});
return result;
}
}
private async Task<Possible<AbsolutePath[]>> TryDownloadNugetAsync(INugetConfiguration configuration, AbsolutePath targetFolder)
{
configuration = configuration ?? new NugetConfiguration();
var downloads = new Task<Possible<ContentHash>>[1 + configuration.CredentialProviders.Count];
var paths = new AbsolutePath[downloads.Length];
var nugetTargetLocation = targetFolder.Combine(m_context.PathTable, "nuget.exe");
var nugetLocation = configuration.ToolUrl;
if (string.IsNullOrEmpty(nugetLocation))
{
var version = configuration.Version ?? "latest";
nugetLocation = string.Format(CultureInfo.InvariantCulture, "https://dist.nuget.org/win-x86-commandline/{0}/nuget.exe", version);
}
TryGetExpectedContentHash(configuration, out var expectedHash);
downloads[0] = m_host.DownloadFile(nugetLocation, nugetTargetLocation, expectedHash, NugetResolverName);
paths[0] = nugetTargetLocation;
for (var i = 0; i < configuration.CredentialProviders.Count; i++)
{
var credentialProvider = configuration.CredentialProviders[i];
var credentialProviderName = NugetResolverName + ".credentialProvider." + i.ToString(CultureInfo.InvariantCulture);
TryGetExpectedContentHash(credentialProvider, out var expectedProviderHash);
var toolUrl = credentialProvider.ToolUrl;
if (string.IsNullOrEmpty(toolUrl))
{
// TODO: Have better provenance for configuration values.
Logger.Log.CredentialProviderRequiresToolUrl(m_context.LoggingContext, credentialProviderName);
return new NugetFailure(NugetFailure.FailureType.FetchCredentialProvider);
}
var fileNameStart = toolUrl.LastIndexOfAny(new[] { '/', '\\' });
var fileName = fileNameStart >= 0 ? toolUrl.Substring(fileNameStart + 1) : toolUrl;
var targetLocation = targetFolder.Combine(m_context.PathTable, fileName);
downloads[i + 1] = m_host.DownloadFile(toolUrl, targetLocation, expectedProviderHash, credentialProviderName);
paths[i + 1] = targetLocation;
}
var results = await Task.WhenAll(downloads);
foreach (var result in results)
{
if (!result.Succeeded)
{
return result.Failure;
}
}
return paths;
}
/// <nodoc />
private bool TryGetExpectedContentHash(IArtifactLocation artifactLocation, out ContentHash? expectedHash)
{
expectedHash = null;
if (!string.IsNullOrEmpty(artifactLocation.Hash))
{
if (!ContentHash.TryParse(artifactLocation.Hash, out var contentHash))
{
// TODO: better provenance for configuration settings.
Logger.Log.NugetDownloadInvalidHash(
m_context.LoggingContext,
"nuget.exe",
artifactLocation.Hash);
return false;
}
expectedHash = contentHash;
}
return true;
}
/// <nodoc />
private sealed class NugetFileStorage : ISandboxedProcessFileStorage
{
private readonly string m_directory;
/// <nodoc />
public NugetFileStorage(string directory)
{
m_directory = directory;
}
/// <inheritdoc />
public string GetFileName(SandboxedProcessFile file)
{
return Path.Combine(m_directory, file.DefaultFileName());
}
}
/// <summary>
/// Helper class that holds all folders required for a nuget resolver
/// </summary>
@ -1753,50 +1282,30 @@ namespace BuildXL.FrontEnd.Nuget
{
private readonly NugetResolverOutputLayout m_resolverLayout;
public NugetPackageOutputLayout(PathTable pathTable, INugetPackage package, AbsolutePath nugetTool, AbsolutePath nugetConfig, NugetResolverOutputLayout resolverLayout)
public NugetPackageOutputLayout(PathTable pathTable, INugetPackage package, NugetResolverOutputLayout resolverLayout)
{
m_resolverLayout = resolverLayout;
Package = package;
NugetTool = nugetTool;
NugetConfig = nugetConfig;
var idAndVersion = package.Id + "." + package.Version;
NugetToolExeConfig = nugetTool.ChangeExtension(pathTable, PathAtom.Create(pathTable.StringTable, ".exe.config"));
// All the folders should include version to avoid potential race conditions during nuget execution.
PackageFolder = PackageRootFolder.Combine(pathTable, idAndVersion);
PackageDirectory = PackageFolder.ToString(pathTable);
PackageTmpDirectory = TempDirectory.Combine(pathTable, idAndVersion).ToString(pathTable);
PackagesConfigFile = ConfigRootFolder.Combine(pathTable, idAndVersion).Combine(pathTable, "packages.config");
PathToNuspec = PackageFolder.Combine(pathTable, $"{package.Id}.nuspec");
}
public static NugetPackageOutputLayout Create(PathTable pathTable, INugetPackage package,
AbsolutePath nugetTool, AbsolutePath nugetConfig, NugetResolverOutputLayout resolverLayout)
NugetResolverOutputLayout resolverLayout)
{
return new NugetPackageOutputLayout(pathTable, package, nugetTool, nugetConfig, resolverLayout);
return new NugetPackageOutputLayout(pathTable, package, resolverLayout);
}
public INugetPackage Package { get; }
public AbsolutePath ResolverFolder => m_resolverLayout.ResolverFolder;
public string ResolverDirectory => m_resolverLayout.ResolverDirectory;
public AbsolutePath NugetTool { get; }
public AbsolutePath NugetToolExeConfig { get; }
public AbsolutePath NugetConfig { get; }
public AbsolutePath TempDirectory => m_resolverLayout.TempDirectory;
public string TempDirectoryAsString => m_resolverLayout.TempDirectoryAsString;
public AbsolutePath PackageRootFolder => m_resolverLayout.PackageRootFolder;
public AbsolutePath ConfigRootFolder => m_resolverLayout.ConfigRootFolder;
public AbsolutePath PackagesConfigFile { get; }
public AbsolutePath PathToNuspec { get; }
public AbsolutePath PackageFolder { get; }

9
Public/Src/FrontEnd/Script/Ambients/Transformers/AmbientTransformer.Process.cs
Просмотреть файл

@ -181,6 +181,8 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
private SymbolAtom m_semaphoreInfoLimit;
private SymbolAtom m_semaphoreInfoName;
private SymbolAtom m_semaphoreInfoIncrementBy;
// Explicitly not exposed in DScript since bypassing the salts is not officially supported, and only used for internally scheduled pips
private SymbolAtom m_unsafeBypassFingerprintSalt;
private CallSignature ExecuteSignature => CreateSignature(
required: RequiredParameters(AmbientTypes.ExecuteArgumentsType),
@ -332,6 +334,7 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
m_unsafeTrustStaticallyDeclaredAccesses = Symbol("trustStaticallyDeclaredAccesses");
m_unsafeDisableFullReparsePointResolving = Symbol("disableFullReparsePointResolving");
m_unsafeDisableSandboxing = Symbol("disableSandboxing");
m_unsafeBypassFingerprintSalt = Symbol("bypassFingerprintSalt");
// Semaphore info.
m_semaphoreInfoLimit = Symbol("limit");
@ -1416,6 +1419,12 @@ namespace BuildXL.FrontEnd.Script.Ambients.Transformers
{
processBuilder.Options |= Process.Options.DisableSandboxing;
}
// UnsafeExecuteArguments.bypassFingerprintSalt
if (Converter.ExtractOptionalBoolean(unsafeOptionsObjLit, m_unsafeBypassFingerprintSalt) == true)
{
processBuilder.Options |= Process.Options.BypassFingerprintSalt;
}
}
private PipId InterpretFinalizationPipArguments(Context context, ObjectLiteral obj)

5
Public/Src/FrontEnd/Sdk/FrontEndEngineAbstraction.cs
Просмотреть файл

@ -222,6 +222,11 @@ namespace BuildXL.FrontEnd.Sdk
return EnumerateEntries(path, pattern, recursive, directories: true);
}
/// <summary>
/// Translates an absolute path according to the defined translations
/// </summary>
public abstract AbsolutePath Translate(AbsolutePath path);
/// <summary>
/// A simple IO based file system entry enumeration helper
/// </summary>

1
Public/Src/FrontEnd/Sdk/FrontEndHost.cs
Просмотреть файл

@ -121,6 +121,7 @@ namespace BuildXL.FrontEnd.Sdk
string weakPackageFingerprint,
PackageIdentity package,
AbsolutePath packageTargetFolder,
AbsolutePath pathToNuspec,
Func<Task<Possible<IReadOnlyList<RelativePath>>>> producePackage);
/// <summary>

8
Public/Src/FrontEnd/Sdk/SimpleFrontEndEngineAbstraction.cs
Просмотреть файл

@ -65,6 +65,12 @@ namespace BuildXL.FrontEnd.Sdk
};
}
/// <inheritdoc/>
public override AbsolutePath Translate(AbsolutePath path)
{
return path;
}
/// <summary>
/// This method is not implemented.
/// </summary>
@ -86,7 +92,7 @@ namespace BuildXL.FrontEnd.Sdk
return false;
}
stream = m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream;
stream = m_fileSystem.OpenText(AbsolutePath.Create(m_pathTable, physicalPath)).BaseStream;
return true;
}

6
Public/Src/FrontEnd/SdkTesting/Helper/TestEngineAbstraction.cs
Просмотреть файл

@ -42,6 +42,12 @@ namespace BuildXL.FrontEnd.Script.Testing.Helper
};
}
/// <inheritdoc/>
public override AbsolutePath Translate(AbsolutePath path)
{
return path;
}
/// <inheritdoc />
public override bool TryGetFrontEndFile(AbsolutePath path, string frontEnd, out Stream stream)
{

3
Public/Src/FrontEnd/UnitTests/Core/DsTest.cs
Просмотреть файл

@ -30,6 +30,7 @@ using BuildXL.FrontEnd.Workspaces.Core;
using BuildXL.Native.IO;
using BuildXL.Pips.Filter;
using BuildXL.Utilities;
using BuildXL.Utilities.Collections;
using BuildXL.Utilities.Configuration;
using BuildXL.Utilities.Configuration.Mutable;
using BuildXL.Utilities.Instrumentation.Common;
@ -1074,6 +1075,8 @@ namespace Test.BuildXL.FrontEnd.Core
frontEndFactory.InitializeFrontEnds(controller, FrontEndContext, engine.Configuration);
var frontEndEngineAbstraction = new BasicFrontEndEngineAbstraction(FrontEndContext.PathTable, engineContext.FileSystem, engine.Configuration);
frontEndEngineAbstraction.TryPopulateWithDefaultMountsTable(LoggingContext, engineContext, engine.Configuration, CollectionUtilities.EmptyDictionary<string, string>());
controller.SetState(frontEndEngineAbstraction, GetPipGraph(), config);
var evaluationFilter = fileToProcess.IsValid ? EvaluationFilter.FromSingleSpecPath(FrontEndContext.SymbolTable, FrontEndContext.PathTable, fileToProcess) : EvaluationFilter.Empty;

80
Public/Src/FrontEnd/UnitTests/Nuget/NuSpecGeneratorTests.cs
Просмотреть файл

@ -11,17 +11,18 @@ using Xunit;
using Xunit.Abstractions;
using Test.BuildXL.TestUtilities.Xunit;
using System;
using BuildXL.Utilities;
namespace Test.BuildXL.FrontEnd.Nuget
{
public class NuSpecGeneratorTests
{
private const int CurrentSpecGenVersion = 11;
private const int CurrentSpecGenVersion = 12;
private readonly ITestOutputHelper m_output;
private readonly FrontEndContext m_context;
private readonly PackageGenerator m_packageGenerator;
private readonly Dictionary<string, string> m_repositories;
private static readonly INugetPackage s_myPackage = new NugetPackage() { Id = "MyPkg", Version = "1.99" };
private static readonly INugetPackage s_systemCollections = new NugetPackage() { Id = "System.Collections", Version = "4.0.11" };
private static readonly INugetPackage s_systemCollectionsConcurrent = new NugetPackage() { Id = "System.Collections.Concurrent", Version = "4.0.12" };
@ -42,6 +43,8 @@ namespace Test.BuildXL.FrontEnd.Nuget
var monikers = new NugetFrameworkMonikers(m_context.StringTable);
m_packageGenerator = new PackageGenerator(m_context, monikers);
m_repositories = new Dictionary<string, string>() { ["BuildXL"] = "https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json" };
}
[Fact]
@ -70,11 +73,11 @@ namespace Test.BuildXL.FrontEnd.Nuget
</package>",
s_packagesOnConfig, new string[] { "lib/net45/my.dll", "lib/net451/my.dll", "lib/netstandard2.0/my.dll"});
var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg);
var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg);
var text = spec.ToDisplayStringV2();
m_output.WriteLine(text);
string expectedSpec = $@"import {{Transformer}} from ""Sdk.Transformers"";
string expectedSpec = $@"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader"";
import * as Managed from ""Sdk.Managed"";
export declare const qualifier: {{
@ -82,22 +85,23 @@ export declare const qualifier: {{
targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"",
}};
const packageRoot = Contents.packageRoot;
namespace Contents {{
export declare const qualifier: {{
}};
export const packageRoot = d`../../../pkgs/TestPkg.1.999`;
const outputDir: Directory = Context.getNewOutputDirectory(""nuget"");
@@public
export const all: StaticDirectory = Transformer.sealDirectory(
packageRoot,
[
f`${{packageRoot}}/lib/net45/my.dll`,
f`${{packageRoot}}/lib/net451/my.dll`,
f`${{packageRoot}}/lib/netstandard2.0/my.dll`,
f`${{packageRoot}}/TestPkg.nuspec`,
]
);
export const all: StaticDirectory = NugetDownloader.downloadPackage({{
id: ""TestPkg"",
version: ""1.999"",
downloadDirectory: outputDir,
extractedFiles: [
r`TestPkg.nuspec`,
r`lib/net45/my.dll`,
r`lib/net451/my.dll`,
r`lib/netstandard2.0/my.dll`,
],
repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]],
}});
}}
@@public
@ -108,9 +112,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
""TestPkg"",
""1.999"",
Contents.all,
[Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)],
[Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net45/my.dll`)],
[importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg]
[Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))],
[Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net45/my.dll`))],
[
importFrom(""System.Collections"").pkg,
importFrom(""System.Collections.Concurrent"").pkg,
]
);
case ""net451"":
case ""net452"":
@ -122,9 +129,12 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
""TestPkg"",
""1.999"",
Contents.all,
[Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)],
[Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/net451/my.dll`)],
[importFrom(""System.Collections"").pkg, importFrom(""System.Collections.Concurrent"").pkg]
[Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))],
[Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/net451/my.dll`))],
[
importFrom(""System.Collections"").pkg,
importFrom(""System.Collections.Concurrent"").pkg,
]
);
case ""netstandard2.0"":
case ""netcoreapp2.0"":
@ -139,9 +149,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
""TestPkg"",
""1.999"",
Contents.all,
[Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`)],
[Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`))],
[
Managed.Factory.createBinaryFromFiles(f`${{packageRoot}}/lib/netstandard2.0/my.dll`),
Managed.Factory.createBinaryFromFiles(Contents.all.getFile(r`lib/netstandard2.0/my.dll`)),
],
[...addIfLazy(qualifier.targetFramework === ""netstandard2.0"", () => [importFrom(""Newtonsoft.Json"").pkg])]
);
@ -150,9 +160,9 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
}};
}}
)();";
XAssert.AreEqual(expectedSpec, text);
XAssert.AreEqual(expectedSpec.Trim(), text.Trim());
const string CurrentSpecHash = "FDBA16F722AB64B5C61F4CBAD40500C9B1944E39";
const string CurrentSpecHash = "1291F51E8E59ED9C2F967C01D49CC39FE6DEB9D8";
ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash);
}
@ -160,25 +170,29 @@ export const pkg: Managed.ManagedNugetPackage = (() => {{
public void GenerateNuSpecForStub()
{
var pkg = m_packageGenerator.AnalyzePackageStub(s_packagesOnConfig);
var spec = new NugetSpecGenerator(m_context.PathTable, pkg).CreateScriptSourceFile(pkg);
var spec = new NugetSpecGenerator(m_context.PathTable, pkg, m_repositories, AbsolutePath.Invalid).CreateScriptSourceFile(pkg);
var text = spec.ToDisplayStringV2();
m_output.WriteLine(text);
string expectedSpec = @"import {Transformer} from ""Sdk.Transformers"";
string expectedSpec = @"import * as NugetDownloader from ""BuildXL.Tools.NugetDownloader"";
export declare const qualifier: {
targetFramework: ""net10"" | ""net11"" | ""net20"" | ""net35"" | ""net40"" | ""net45"" | ""net451"" | ""net452"" | ""net46"" | ""net461"" | ""net462"" | ""net472"" | ""netstandard1.0"" | ""netstandard1.1"" | ""netstandard1.2"" | ""netstandard1.3"" | ""netstandard1.4"" | ""netstandard1.5"" | ""netstandard1.6"" | ""netstandard2.0"" | ""netcoreapp2.0"" | ""netcoreapp2.1"" | ""netcoreapp2.2"" | ""netcoreapp3.0"" | ""netcoreapp3.1"" | ""net5.0"" | ""net6.0"" | ""netstandard2.1"",
targetRuntime: ""win-x64"" | ""osx-x64"" | ""linux-x64"",
};
const packageRoot = Contents.packageRoot;
namespace Contents {
export declare const qualifier: {
};
export const packageRoot = d`../../../pkgs/TestPkgStub.1.999`;
const outputDir: Directory = Context.getNewOutputDirectory(""nuget"");
@@public
export const all: StaticDirectory = Transformer.sealDirectory(packageRoot, []);
export const all: StaticDirectory = NugetDownloader.downloadPackage({
id: ""TestPkgStub"",
version: ""1.999"",
downloadDirectory: outputDir,
extractedFiles: [],
repositories: [[""BuildXL"", ""https://pkgs.dev.azure.com/cloudbuild/_packaging/BuildXL.Selfhost/nuget/v3/index.json""]],
});
}
@@public
@ -189,7 +203,7 @@ export const pkg: NugetPackage = {
};";
XAssert.ArrayEqual(SplitToLines(expectedSpec), SplitToLines(text));
const string CurrentSpecHash = "8550BF2B35888E3C0095095F44E2EB06FC2F0576";
const string CurrentSpecHash = "389BC336BD4B206394E01B1E76A859E6561CE30D";
ValidateCurrentSpecGenVersion(expectedSpec, CurrentSpecHash);
}

16
Public/Src/FrontEnd/UnitTests/Nuget/NugetResolverUnitTests.cs
Просмотреть файл

@ -64,7 +64,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
var nugetResolver = CreateWorkspaceNugetResolverForTesting();
var allPackages = new Dictionary<string, INugetPackage> { [packageOnDisk.Package.Id] = packageOnDisk.Package };
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
XAssert.IsTrue(analyzedPackage.Succeeded);
@ -86,7 +86,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
var allPackages = new Dictionary<string, INugetPackage> { [packageOnDisk.Package.Id] = packageOnDisk.Package };
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
XAssert.IsTrue(analyzedPackage.Succeeded);
@ -105,10 +105,10 @@ namespace Test.BuildXL.FrontEnd.Nuget
var packageText = File.ReadAllText(packageDsc);
XAssert.IsFalse(packageText.Contains("export * from"));
XAssert.IsFalse(packageText.Contains("/Foo.Bar.dsc\""));
XAssert.IsTrue(packageText.Contains("/pkgs/Foo.Bar.1.2`"));
XAssert.IsTrue(packageText.Contains("packageRoot = d`"));
XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/a.txt`"));
XAssert.IsTrue(packageText.Contains("f`${packageRoot}/Folder/b.txt`"));
XAssert.IsTrue(packageText.Contains(@"id: ""Foo.Bar"""));
XAssert.IsTrue(packageText.Contains(@"version: ""1.2"""));
XAssert.IsTrue(packageText.Contains("r`Folder/a.txt`"));
XAssert.IsTrue(packageText.Contains("r`Folder/b.txt`"));
}
[Fact]
@ -140,7 +140,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
var pathTable = m_testContext.PathTable;
var packageOnDisk = CreateTestPackageOnDisk(includeScriptSpec: false, version: version);
var nugetResolver = CreateWorkspaceNugetResolverForTesting();
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, false);
var analyzedPackage = nugetResolver.AnalyzeNugetPackage(packageOnDisk, AbsolutePath.Invalid, false);
XAssert.IsTrue(analyzedPackage.Succeeded);
var allPackages = new Dictionary<string, NugetAnalyzedPackage> { [packageOnDisk.Package.Id] = analyzedPackage.Result };
XAssert.IsTrue(analyzedPackage.Succeeded);
@ -314,7 +314,7 @@ $@"module({{
{
var allPackages = packagesOnDisk.ToDictionary(kvp => kvp.Package.Id, kvp => kvp.Package);
var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, false).Result);
var allAnalyzedPackages = packagesOnDisk.ToDictionary(package => package.Package.Id, package => resolver.AnalyzeNugetPackage(package, AbsolutePath.Invalid, false).Result);
resolver.SetDownloadedPackagesForTesting(allAnalyzedPackages);

4
Public/Src/FrontEnd/UnitTests/Nuget/PackageGenerator.cs
Просмотреть файл

@ -44,7 +44,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
paths,
"testPackageHash"));
return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false);
return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, XDocument.Parse(xml), packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid);
}
public NugetAnalyzedPackage AnalyzePackageStub(Dictionary<string, INugetPackage> packagesOnConfig)
@ -58,7 +58,7 @@ namespace Test.BuildXL.FrontEnd.Nuget
new PackageIdentity("nuget", nugetPackage.Id, nugetPackage.Version, nugetPackage.Alias),
packageFolder: AbsolutePath.Create(m_context.PathTable, X("/X/Pkgs/TestPkgStub/1.999"))));
return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false);
return NugetAnalyzedPackage.TryAnalyzeNugetPackage(m_context, m_monikers, nuSpec: null, packageOnDisk, packagesOnConfig, false, AbsolutePath.Invalid);
}
}
}

12
Public/Src/FrontEnd/UnitTests/Script.Workspaces.Utilities/FakeNugetResolverSettingsProvider.cs
Просмотреть файл

@ -86,18 +86,6 @@ namespace Test.DScript.Workspaces.Utilities
{
return new MalformedConfigurationFailure("An array literal is expected for NuGet credential providers");
}
var credentialProviders = expression.Cast<IArrayLiteralExpression>();
foreach (var element in credentialProviders.Elements)
{
var elementResult = ParseNugetConfigurationFrom(element.As<IObjectLiteralExpression>());
if (!elementResult.Succeeded)
{
return elementResult;
}
result.CredentialProviders.Add(elementResult.Result);
}
}
if (configurationExpression.TryFindAssignmentPropertyInitializer(NaiveConfigurationParsingUtilities.ToolUrlFieldName, out expression))

3
Public/Src/IDE/UnitTests/LanguageServer/TestAutoCompleteProvider.cs
Просмотреть файл

@ -145,11 +145,12 @@ namespace BuildXL.Ide.LanguageServer.UnitTests
.OrderBy(ci => ci.Label)
.ToArray();
Assert.Equal(1, completionItems.Length);
Assert.Equal(2, completionItems.Length);
// TODO: Change to just BuildXL.DScript check after April 15, 2019 when deployed bits have updated.
Assert.True(completionItems[0].Label == "BuildXL.DScript.LanguageServer.UnitTests.Data.Module" ||
completionItems[0].Label == "BuildXLScript.LanguageServer.UnitTests.Data.Module");
Assert.True(completionItems[1].Label == "BuildXL.Tools.NugetDownloader");
}
[Fact]

11
Public/Src/Pips/Dll/Graph/ExtraFingerprintSalts.cs
Просмотреть файл

@ -413,9 +413,9 @@ namespace BuildXL.Pips.Graph
/// <summary>
/// Add fields to fingerprint
/// </summary>
public void AddFingerprint(IHashingHelper fingerprinter)
public void AddFingerprint(IHashingHelper fingerprinter, bool bypassFingerprintSaltAndVersion)
{
if (!string.IsNullOrEmpty(FingerprintSalt))
if (!bypassFingerprintSaltAndVersion && !string.IsNullOrEmpty(FingerprintSalt))
{
fingerprinter.Add(nameof(FingerprintSalt), FingerprintSalt);
}
@ -455,14 +455,17 @@ namespace BuildXL.Pips.Graph
fingerprinter.Add(nameof(ExplicitlyReportDirectoryProbes), 1);
}
fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion);
if (!bypassFingerprintSaltAndVersion)
{
fingerprinter.Add(nameof(FingerprintVersion), (int)FingerprintVersion);
}
}
private CalculatedFingerprintTuple ComputeWeakFingerprint()
{
using (var hasher = new CoreHashingHelper(true))
{
AddFingerprint(hasher);
AddFingerprint(hasher, bypassFingerprintSaltAndVersion: false);
return new CalculatedFingerprintTuple(hasher.GenerateHash(), hasher.FingerprintInputText);
}
}

2
Public/Src/Pips/Dll/Graph/PipFingerprinter.cs
Просмотреть файл

@ -173,7 +173,7 @@ namespace BuildXL.Pips.Graph
Contract.Requires(fingerprinter != null);
Contract.Requires(pip != null);
fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp));
fingerprinter.AddNested(PipFingerprintField.ExecutionAndFingerprintOptions, fp => m_extraFingerprintSalts.AddFingerprint(fp, pip.BypassFingerprintSalt));
// Fingerprints must change when outputs are hashed with a different algorithm.
fingerprinter.Add(PipFingerprintField.ContentHashAlgorithmName, s_outputContentHashAlgorithmName);

5
Public/Src/Pips/Dll/Operations/Pip.cs
Просмотреть файл

@ -70,6 +70,11 @@ namespace BuildXL.Pips.Operations
[PipCaching(FingerprintingRole = FingerprintingRole.None)]
public abstract PipType PipType { get; }
/// <summary>
/// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both EngineEnvironmentSettings.DebugFingerprintSalt and PipFingerprintingVersion.TwoPhaseV2
/// </summary>
public virtual bool BypassFingerprintSalt => false;
/// <nodoc />
internal Pip()
{

5
Public/Src/Pips/Dll/Operations/Process.Options.cs
Просмотреть файл

@ -182,6 +182,11 @@ namespace BuildXL.Pips.Operations
/// Whether to disable sandboxing for this process
/// </summary>
DisableSandboxing = 1 << 21,
/// <summary>
/// When set, the pip fingerprint is not sensitive to fingerprint salts. This excludes both <see cref="EngineEnvironmentSettings.DebugFingerprintSalt"/> and PipFingerprintingVersion.TwoPhaseV2
/// </summary>
BypassFingerprintSalt = 1 << 22,
}
}
}

4
Public/Src/Pips/Dll/Operations/Process.cs
Просмотреть файл

@ -842,6 +842,10 @@ namespace BuildXL.Pips.Operations
/// </summary>
public bool HasSharedOpaqueDirectoryOutputs => DirectoryOutputs.Any(d => d.IsSharedOpaque);
/// <inheritdoc/>
[PipCaching(FingerprintingRole = FingerprintingRole.Semantic)]
public override bool BypassFingerprintSalt => (ProcessOptions & Options.BypassFingerprintSalt) != 0;
/// <summary>
/// What policy to apply when merging redirected outputs back
/// </summary>

38
Public/Src/Sandbox/MacOs/deployment.dsc
Просмотреть файл

@ -7,8 +7,6 @@ import * as SdkDeployment from "Sdk.Deployment";
namespace Deployment {
export declare const qualifier: {configuration: "debug" | "release"};
const pkgPath = d`${importFrom("runtime.osx-x64.BuildXL").Contents.all.root}/runtimes/osx-x64/native`;
@@public
export const macBinaryUsage = Context.getCurrentHost().os === "macOS"
? (BuildXLSdk.Flags.isValidatingOsxRuntime ? "package" : "build")
@ -22,7 +20,9 @@ namespace Deployment {
contents: macBinaryUsage === "none"
? []
: macBinaryUsage === "package"
? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext`) ]
? [ SdkDeployment.createFromFilteredStaticDirectory(
importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext`}),
r`.`)]
: [{
subfolder: a`Contents`,
contents: [
@ -47,7 +47,9 @@ namespace Deployment {
contents: macBinaryUsage === "none"
? []
: macBinaryUsage === "package"
? [ SdkDeployment.createFromDisk(d`${pkgPath}/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`) ]
? [ SdkDeployment.createFromFilteredStaticDirectory(
importFrom("runtime.osx-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/osx-x64/native/${qualifier.configuration}/BuildXLSandbox.kext.dSYM`}),
r`.`)]
: [{
subfolder: a`Contents`,
contents: [
@ -83,30 +85,34 @@ namespace Deployment {
@@public
export const sandboxMonitor: SdkDeployment.Definition = {
contents: [
contents:
macBinaryUsage === "build"
? Sandbox.monitor
: f`${pkgPath}/${qualifier.configuration}/SandboxMonitor`
]
? [Sandbox.monitor]
: macBinaryUsage === "package"
? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/SandboxMonitor`)]
: []
};
@@public
export const interopLibrary: SdkDeployment.Definition = {
contents: macBinaryUsage === "build"
? [ Sandbox.libInterop, Sandbox.libDetours ]
: [
f`${pkgPath}/${qualifier.configuration}/libBuildXLInterop.dylib`,
f`${pkgPath}/${qualifier.configuration}/libBuildXLDetours.dylib`
]
: macBinaryUsage === "package"
? [
importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLInterop.dylib`),
importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/libBuildXLDetours.dylib`)
]
: []
};
@@public
export const coreDumpTester: SdkDeployment.Definition = {
contents: [
contents:
macBinaryUsage === "build"
? Sandbox.coreDumpTester
: f`${pkgPath}/${qualifier.configuration}/CoreDumpTester`
]
? [Sandbox.coreDumpTester]
: macBinaryUsage === "package"
? [importFrom("runtime.osx-x64.BuildXL").Contents.all.getFile(r`runtimes/osx-x64/native/${qualifier.configuration}/CoreDumpTester`)]
: []
};
@@public

2
Public/Src/Tools/BxlScriptAnalyzer/WorkspaceBuilder.cs
Просмотреть файл

@ -218,7 +218,7 @@ namespace BuildXL.FrontEnd.Script.Analyzer
Contract.AssertNotNull(frontEndEngineAbstraction);
AddConfigurationMounts(config, mountsTable);
languageServiceEngine.SetMountsTable(mountsTable);
languageServiceEngine.UpdateMountsTable(mountsTable);
}
if (frontEndEngineAbstraction == null)

188
Public/Src/Tools/FileDownloader/Downloader.cs
Просмотреть файл

@ -2,21 +2,19 @@
// Licensed under the MIT License.
using System;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Native.IO;
using BuildXL.Storage;
using BuildXL.ToolSupport;
using BuildXL.Utilities;
using Microsoft.IdentityModel.Clients.ActiveDirectory;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using BuildXL.Utilities.VstsAuthentication;
namespace Tool.Download
{
@ -94,11 +92,15 @@ namespace Tool.Download
var httpRequest = new HttpRequestMessage(HttpMethod.Get, arguments.Url);
var logger = new StringBuilder();
// If the download URI is pointing to a VSTS feed and we get a valid auth token, make it part of the request
// We only want to send the token over HTTPS and to a VSTS domain to avoid security issues
if (IsVSTSPackageSecureURI(arguments.Url) &&
await TryGetAuthenticationHeaderAsync(arguments.Url) is var authHeader &&
authHeader != null)
if (VSTSAuthenticationHelper.IsVSTSPackageSecureURI(arguments.Url) &&
await VSTSAuthenticationHelper.IsAuthenticationRequiredAsync(arguments.Url, CancellationToken.None, logger) &&
await VSTSAuthenticationHelper.TryGetAuthenticationCredentialsAsync(arguments.Url, CancellationToken.None) is var maybeAuthCredentials &&
maybeAuthCredentials.Succeeded &&
VSTSAuthenticationHelper.GetAuthenticationHeaderFromPAT(maybeAuthCredentials.Result.pat) is var authHeader)
{
httpRequest.Headers.Accept.Clear();
httpRequest.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
@ -162,175 +164,5 @@ namespace Tool.Download
return await ContentHashingUtilities.HashContentStreamAsync(fs, hashType);
}
private bool IsVSTSPackageSecureURI(Uri downloadURI)
{
return downloadURI.Scheme == "https" &&
(downloadURI.Host.EndsWith(".pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith(".pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase)) ;
}
/// <summary>
/// Tries to authenticate using a credential provider and fallback to IWA if that fails.
/// </summary>
private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderAsync(Uri uri)
{
var result = await TryGetAuthenticationHeaderValueWithCredentialProviderAsync(uri);
if (result != null)
{
return result;
}
return await TryGetAuthenticationHeaderWithIWAAsync(uri);
}
/// <summary>
/// Tries to get an authentication token using Integrated Windows Authentication with the current logged in user
/// </summary>
/// <returns>Null if authentication fails</returns>
/// <remarks>
/// The auth token is acquired to address simple auth cases for retrieving packages from VSTS feeds from Windows domain
/// joined machines where IWA is enabled.
/// See https://github.com/AzureAD/microsoft-authentication-library-for-dotnet/wiki/Integrated-Windows-Authentication
/// </remarks>
private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderWithIWAAsync(Uri uri)
{
var authenticationContext = new AuthenticationContext(m_authority);
try
{
var userCredential = new UserCredential($"{Environment.UserName}@{Tenant}");
// Many times the user UPN cannot be automatically retrieved, so we build it based on the current username and tenant. This might
// not be perfect but should work for most cases. Getting the UPN for a given user from AD requires authentication as well.
var result = await authenticationContext.AcquireTokenAsync(Resource, Client, userCredential); ;
return new AuthenticationHeaderValue("Bearer", result.AccessToken);
}
catch (AdalException ex)
{
Console.WriteLine($"Download resolver was not able to authenticate using Integrated Windows Authentication for '{uri}': {ex.Message}");
// Getting an auth token via IWA is on a best effort basis. If anything fails, we silently continue without auth
return null;
}
}
/// <summary>
/// Tries to get an authentication token using a credential provider
/// </summary>
/// <returns>Null if authentication fails</returns>
/// <remarks>
/// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers
/// using an environment variable.
/// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general
/// </remarks>
private async Task<AuthenticationHeaderValue> TryGetAuthenticationHeaderValueWithCredentialProviderAsync(Uri uri)
{
string credentialProviderPath = DiscoverCredentialProvider(uri);
if (credentialProviderPath == null)
{
// Failure has been logged already
return null;
}
// Call the provider with the requested URI in non-interactive mode.
var processInfo = new ProcessStartInfo
{
FileName = credentialProviderPath,
Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive",
RedirectStandardOutput = true,
CreateNoWindow = true,
};
using (var process = Process.Start(processInfo))
{
try
{
#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
process?.WaitForExit();
#pragma warning restore AsyncFixer02
}
catch (Exception e) when (e is SystemException || e is Win32Exception)
{
ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}");
return null;
}
if (process == null)
{
// The process was not started
ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not start credential provider process '{credentialProviderPath}'.");
return null;
}
// Check whether the authorization succeeded
if (process.ExitCode == 0)
{
try
{
// The response should be a well-formed JSON with a 'password' entry representing the PAT
var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput));
var pat = response.GetValue("Password");
if (pat == null)
{
ReportAuthenticationViaCredentialProviderFailed(uri, $"Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'.");
return null;
}
// We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT
return new AuthenticationHeaderValue("Basic",
Convert.ToBase64String(
System.Text.ASCIIEncoding.ASCII.GetBytes(
string.Format("{0}:{1}", "", pat))));
}
catch (JsonReaderException)
{
ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' response is not a well-formed JSON.");
return null;
}
}
ReportAuthenticationViaCredentialProviderFailed(uri, $"The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}.");
}
return null;
}
private string DiscoverCredentialProvider(Uri uri)
{
var paths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH");
if (paths == null)
{
ReportAuthenticationViaCredentialProviderFailed(uri, $"NUGET_CREDENTIALPROVIDERS_PATH is not set.");
return null;
}
// Here we do something slightly simpler than what NuGet does and just look for the first credential
// provider we can find
string credentialProviderPath = null;
foreach (string path in paths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
credentialProviderPath = Directory.EnumerateFiles(path, "credentialprovider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault();
if (credentialProviderPath != null)
{
break;
}
}
if (credentialProviderPath == null)
{
ReportAuthenticationViaCredentialProviderFailed(uri, $"Credential provider was not found under '{paths}'.");
return null;
}
return credentialProviderPath;
}
private void ReportAuthenticationViaCredentialProviderFailed(Uri url, string details)
{
Console.WriteLine($"Download resolver was not able to authenticate using a credential provider for '{url}': {details}") ;
}
}
}

1
Public/Src/Tools/FileDownloader/Tool.FileDownloader.dsc
Просмотреть файл

@ -21,6 +21,7 @@ namespace FileDownloader {
sources: globR(d`.`, "Downloader*.cs"),
references:[
importFrom("Newtonsoft.Json").pkg,
importFrom("BuildXL.Utilities").VstsAuthentication.dll,
importFrom("BuildXL.Utilities").Collections.dll,
importFrom("BuildXL.Utilities").Native.dll,
importFrom("BuildXL.Utilities.Instrumentation").Common.dll,

6
Public/Src/Tools/NugetDownloader/App.config Normal file
Просмотреть файл

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<runtime>
<ThrowUnobservedTaskExceptions enabled="true"/>
</runtime>
</configuration>

84
Public/Src/Tools/NugetDownloader/ConsoleLogger.cs Normal file
Просмотреть файл

@ -0,0 +1,84 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Threading.Tasks;
using NuGet.Common;
namespace NugetDownloader
{
/// <summary>
/// Logs all messages to the console
/// </summary>
internal class ConsoleLogger : ILogger
{
/// <nodoc/>
public void Log(LogLevel level, string data)
{
Console.WriteLine($"[{level}]:{data}");
}
/// <nodoc/>
public void Log(ILogMessage message)
{
Console.WriteLine(message.FormatWithCode());
}
/// <nodoc/>
public Task LogAsync(LogLevel level, string data)
{
Log(level, data);
return Task.CompletedTask;
}
/// <nodoc/>
public Task LogAsync(ILogMessage message)
{
Log(message);
return Task.CompletedTask;
}
/// <nodoc/>
public void LogDebug(string data)
{
Log(LogLevel.Debug, data);
}
/// <nodoc/>
public void LogError(string data)
{
Log(LogLevel.Error, data);
}
/// <nodoc/>
public void LogInformation(string data)
{
Log(LogLevel.Information, data);
}
/// <nodoc/>
public void LogInformationSummary(string data)
{
Log(LogLevel.Information, data);
}
/// <nodoc/>
public void LogMinimal(string data)
{
Log(LogLevel.Information, data);
}
/// <nodoc/>
public void LogVerbose(string data)
{
Log(LogLevel.Verbose, data);
}
/// <nodoc/>
public void LogWarning(string data)
{
Log(LogLevel.Warning, data);
}
}
}

119
Public/Src/Tools/NugetDownloader/LiteralFiles/Tool.NugetDownloader.dsc.literal Normal file
Просмотреть файл

@ -0,0 +1,119 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
import {Artifact, Cmd, Transformer, Tool} from "Sdk.Transformers";
export declare const qualifier: {};
const downloaderDefinition: Transformer.ToolDefinition = {
// CODESYNC: keep in sync with deployment at Public\Src\FrontEnd\Nuget\BuildXL.FrontEnd.Nuget.dsc
exe: Context.isWindowsOS()? f`${Context.getBuildEngineDirectory()}/NugetDownloader.exe` : f`${Context.getBuildEngineDirectory()}/NugetDownloader`,
description: "BuildXL NuGet downloader",
dependsOnWindowsDirectories: true,
dependsOnAppDataDirectory: true,
prepareTempDirectory: true,
untrackedDirectoryScopes: [
d`${Context.getMount("ProgramData").path}/Microsoft/NetFramework/BreadcrumbStore`,
...addIfLazy(Context.isWindowsOS(), () => [d`${Context.getMount("LocalLow").path}/Microsoft/CryptnetUrlCache`]),
],
};
/**
* Arguments for downloading a NuGet package under a specified directory
*/
@@public
export interface Arguments extends Transformer.RunnerArguments {
id: string;
version: string;
downloadDirectory: Directory;
extractedFiles: RelativePath[];
repositories: [string, string][];
credentialProviderPath?: File;
timeoutInMinutes?: number;
}
/**
* Downloads a NuGet package as specified in the arguments and returns the outputs that result of the execution in a static directory
*/
@@public
export function downloadPackage(args: Arguments) : PartialStaticContentDirectory {
const arguments: Argument[] = [
Cmd.option("/id:", args.id),
Cmd.option("/version:", args.version),
Cmd.option("/downloadDirectory:", Artifact.none(args.downloadDirectory)),
Cmd.options("/repositories:", args.repositories.map(kvp => kvp[0] + "=" + kvp[1])),
];
// NuGet extraction always injects a parent folder with the package id + version as the name
const directoryName = `${args.id}.${args.version}`;
// Build the list of expected outputs
const outputs : File[] = args.extractedFiles.map(relativePath => f`${args.downloadDirectory}/${directoryName}/${relativePath}`);
// In some cases the package is expected to be empty. In that case, don't run anything and just return an empty seal dir
if (outputs.length === 0)
{
return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, []);
}
let tool = downloaderDefinition;
// The timeout is not available per-pip but at the tool level
if (args.timeoutInMinutes !== undefined) {
tool = tool.merge<Transformer.ToolDefinition>({
timeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000,
// We could have a separate argument for this, but for now let's keep it simple
warningTimeoutInMilliseconds: args.timeoutInMinutes * 60 * 1000
});
}
// Nuget tries to read the user settings config file, and if it is not there, it creates it.
// The creation does not behave well with concurrent executions and write locks issues arise.
// Let's give each nuget pip a different app data directory
const redirectedAppData = Context.getTempDirectory("redirectedAppData");
const transformerExecuteArgs : Transformer.ExecuteArguments = {
description: `Downloading NuGet package '${args.id}.${args.version}'`,
tool: tool,
arguments: arguments,
workingDirectory: args.downloadDirectory,
tags: ["nuget", ...(args.tags || [])],
environmentVariables: [
...addIf(args.credentialProviderPath !== undefined, {name: "NUGET_CREDENTIALPROVIDERS_PATH", value: args.credentialProviderPath}),
...addIf(Environment.hasVariable("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS"), {name: "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", value: Environment.getStringValue("VSS_NUGET_EXTERNAL_FEED_ENDPOINTS")}),
{name: "AppData", value: redirectedAppData}
],
outputs: outputs,
tempDirectory: redirectedAppData,
unsafe: <Transformer.UnsafeExecuteArguments>{
passThroughEnvironmentVariables: [
"LocalAppData",
"QAUTHMATERIALROOT",
"__CREDENTIAL_PROVIDER_LOG_DIR",
"__CLOUDBUILD_AUTH_HELPER_ROOT__",
"__CLOUDBUILD_AUTH_HELPER_CONFIG__"
],
requireGlobalDependencies: false,
// The only child process this tool spawns is the credential provider process. This process may write log files that are hard to predict upfront.
hasUntrackedChildProcesses: true,
// This is to avoid a cache miss on a NuGet download due to changes on the nuget download tool (changes in downtream dependencies for the
// most part, e.g. some framework dll that is changed because of some unrelated change in Bxl)
// This is a confidence vote since we own the nuget downloader tool. If for any reason the nuget pips need invalidation, that can
// be manually triggered by bumping the NuGet spec fingerprint version.
untrackedScopes: [Context.getBuildEngineDirectory()],
// Make NuGet pips so fingerprint salts (the general fingerprint V2 version and the debug salt passed with /p:BUILDXL_FINGERPRINT_SALT) are not
// included in the weak fingerprint. Observe this option is not exposed in DScript, so we are forcing the hand of the type checker with the cast above
// to not complain about this extra field (which will be picked up at runtime)
bypassFingerprintSalt: true
},
// This is to avoid DFAs under the credential provider directory, which is an arbitrarily specified directory that would need to
// be contained in a statically declared read mount otherwise
allowUndeclaredSourceReads: true,
};
const result = Transformer.execute(transformerExecuteArgs);
return Transformer.sealPartialDirectory(d`${args.downloadDirectory.combine(directoryName)}`, result.getOutputFiles());
}

6
Public/Src/Tools/NugetDownloader/LiteralFiles/module.config.dsc.literal Normal file
Просмотреть файл

@ -0,0 +1,6 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
module({
name: "BuildXL.Tools.NugetDownloader"
});

149
Public/Src/Tools/NugetDownloader/NugetDownloader.cs Normal file
Просмотреть файл

@ -0,0 +1,149 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Native.IO;
using BuildXL.ToolSupport;
using BuildXL.Utilities;
using BuildXL.Utilities.VstsAuthentication;
using NuGet.Common;
using NuGet.Configuration;
using NuGet.Packaging;
using NuGet.Packaging.Signing;
using NuGet.Protocol.Core.Types;
using NugetDownloader;
namespace Tool.Download
{
/// <summary>
/// Downloads a Nuget into a given directory from a collection of source repositories
/// </summary>
internal sealed class NugetDownloader : ToolProgram<NugetDownloaderArgs>
{
private NugetDownloader() : base("NugetDownloader")
{
}
/// <nodoc />
public static int Main(string[] arguments)
{
return new NugetDownloader().MainHandler(arguments);
}
/// <inheritdoc />
public override bool TryParse(string[] rawArgs, out NugetDownloaderArgs arguments)
{
try
{
arguments = new NugetDownloaderArgs(rawArgs);
return true;
}
catch (Exception ex)
{
Console.Error.WriteLine(ex.GetLogEventMessage());
arguments = null;
return false;
}
}
/// <inheritdoc />
public override int Run(NugetDownloaderArgs arguments)
{
return TryDownloadNugetToDiskAsync(arguments).GetAwaiter().GetResult();
}
/// <summary>
/// Attempts to download and extract a NuGet package to disk.
/// </summary>
private async Task<int> TryDownloadNugetToDiskAsync(NugetDownloaderArgs arguments)
{
Console.WriteLine($"Download started for package '{arguments.Id}' and version '{arguments.Version}' to '{arguments.DownloadDirectory}'.");
var stopwatch = Stopwatch.StartNew();
SourceCacheContext cache = new SourceCacheContext();
var authLogger = new StringBuilder();
var maybeRepositories = await VSTSAuthenticationHelper.TryCreateSourceRepositories(arguments.Repositories.Select(kvp => (kvp.Key, kvp.Value)), CancellationToken.None, authLogger);
// Informational data
Console.WriteLine(authLogger.ToString());
if (!maybeRepositories.Succeeded)
{
Console.Error.WriteLine($"Failed to access the specified repositories: " + maybeRepositories.Failure.Describe());
return 1;
}
bool found = false;
ILogger logger = new ConsoleLogger();
foreach (var sourceRepository in maybeRepositories.Result)
{
Console.Write($"Finding resource against source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}'. Authentication is on: {sourceRepository.PackageSource.Credentials != null}");
// TODO: maybe we need a retry here? or define a default retry in the DScript SDK?
FindPackageByIdResource resource = await sourceRepository.GetResourceAsync<FindPackageByIdResource>();
if (resource == null)
{
Console.Write($"Resource under source repository '{sourceRepository.PackageSource.Name}={sourceRepository.PackageSource.SourceUri}' not found.");
continue;
}
var destinationFilePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
// We don't really need to materialize the nupkg file on disk, but a memory stream has a 2GB limitation, and NuGet packages
// can be larger than that. Just place it on disk and remove it after we are done extracting.
using (var packageStream = new FileStream(
destinationFilePath,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.ReadWrite | FileShare.Delete,
bufferSize: 4096))
{
if (await resource.CopyNupkgToStreamAsync(
arguments.Id,
arguments.Version,
packageStream,
cache,
logger,
CancellationToken.None))
{
found = true;
await PackageExtractor.ExtractPackageAsync(
sourceRepository.PackageSource.Source,
packageStream,
new PackagePathResolver(arguments.DownloadDirectory),
new PackageExtractionContext(
PackageSaveMode.Files | PackageSaveMode.Nuspec,
XmlDocFileSaveMode.None,
ClientPolicyContext.GetClientPolicy(NullSettings.Instance, logger),
logger),
CancellationToken.None);
FileUtilities.DeleteFile(destinationFilePath);
break;
}
}
}
if (!found)
{
Console.Error.WriteLine($"Could not find NuGet package '{arguments.Id}' with version '{arguments.Version.ToNormalizedString()}' under any of the provided repositories.");
return 1;
}
Console.WriteLine($"Finished downloading package '{arguments.Id}' with version '{arguments.Version}' in {stopwatch.ElapsedMilliseconds}ms");
return 0;
}
}
}

93
Public/Src/Tools/NugetDownloader/NugetDownloaderArgs.cs Normal file
Просмотреть файл

@ -0,0 +1,93 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using BuildXL.ToolSupport;
using NuGet.Versioning;
namespace Tool.Download
{
/// <nodoc/>
internal sealed partial class NugetDownloaderArgs : CommandLineUtilities
{
/// <summary>
/// package id of the download
/// </summary>
public string Id { get; }
/// <summary>
/// package version of the download
/// </summary>
public NuGetVersion Version { get; }
/// <summary>
/// The directory to place the downloaded file
/// </summary>
public string DownloadDirectory { get; }
/// <summary>
/// The Nuget feeds (name and Uri) where to download the package from
/// </summary>
public IReadOnlyDictionary<string, Uri> Repositories { get; }
/// <nodoc />
public NugetDownloaderArgs(string[] args)
: base(args)
{
var repositories = new Dictionary<string, Uri>();
foreach (Option opt in Options)
{
if (opt.Name.Equals("id", StringComparison.OrdinalIgnoreCase))
{
Id = opt.Value;
}
else if (opt.Name.Equals("version", StringComparison.OrdinalIgnoreCase))
{
if (!NuGetVersion.TryParse(opt.Value, out NuGetVersion version))
{
throw Error($"Malformed version: {opt.Value}.");
}
Version = version;
}
else if (opt.Name.Equals("downloadDirectory", StringComparison.OrdinalIgnoreCase))
{
DownloadDirectory = opt.Value;
}
else if (opt.Name.Equals("repositories", StringComparison.OrdinalIgnoreCase))
{
var kvp = ParseKeyValuePair(opt);
repositories[kvp.Key] = new Uri(kvp.Value);
}
else
{
throw Error($"Unsupported option: {opt.Name}.");
}
}
Repositories = repositories;
if (Id == null)
{
throw Error($"Missing mandatory argument 'id'");
}
if (Version == null)
{
throw Error($"Missing mandatory argument 'version'");
}
if (Repositories.Count == 0)
{
throw Error($"Missing mandatory argument 'repositories'");
}
if (string.IsNullOrWhiteSpace(DownloadDirectory))
{
throw Error($"Missing mandatory argument 'DownloadDirectory'");
}
}
}
}

62
Public/Src/Tools/NugetDownloader/Tool.NugetDownloader.dsc Normal file
Просмотреть файл

@ -0,0 +1,62 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
import * as Managed from "Sdk.Managed";
import * as BuildXLSdk from "Sdk.BuildXL";
import {Transformer} from "Sdk.Transformers";
import * as Deployment from "Sdk.Deployment";
import * as MSBuild from "Sdk.Selfhost.MSBuild";
import * as Frameworks from "Sdk.Managed.Frameworks";
import * as Shared from "Sdk.Managed.Shared";
namespace NugetDownloader {
export declare const qualifier : BuildXLSdk.DefaultQualifierWithNet472;
// These are the specs that define the nuget downloader SDK. Since this is a bxl-provided tool
// and customers don't have direct exposure to it, they are not places under the usual SDK folder, but
// deployed as part of bxl binaries
@@public
export const specDeployment: Deployment.Definition = {
contents: [
{subfolder: r`Sdk/Sdk.NugetDownloader`, contents: [
{file: f`LiteralFiles/Tool.NugetDownloader.dsc.literal`, targetFileName: a`Tool.NugetDownloader.dsc`},
{file: f`LiteralFiles/module.config.dsc.literal`, targetFileName: a`module.config.dsc`},]
}
]
};
@@public
export const dll = BuildXLSdk.executable({
assemblyName: "NugetDownloader",
skipDocumentationGeneration: true,
skipDefaultReferences: true,
sources: globR(d`.`, "*.cs"),
references:[
...addIf(BuildXLSdk.isFullFramework,
NetFx.Netstandard.dll
),
importFrom("BuildXL.Utilities").VstsAuthentication.dll,
importFrom("BuildXL.Utilities").ToolSupport.dll,
importFrom("BuildXL.Utilities").Native.dll,
importFrom("BuildXL.Utilities").dll,
importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
],
runtimeContentToSkip:[
importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg,
],
runtimeContent: [
f`App.config`,
specDeployment
],
deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Collections.Immutable.dll`] },
});
}

4
Public/Src/Utilities/Configuration/Resolvers/INugetConfiguration.cs
Просмотреть файл

@ -11,8 +11,8 @@ namespace BuildXL.Utilities.Configuration
public partial interface INugetConfiguration : IArtifactLocation
{
/// <summary>
/// Optional credential helper to use for NuGet
/// The download timeout, in minutes, for each NuGet download pip
/// </summary>
IReadOnlyList<IArtifactLocation> CredentialProviders { get; }
int? DownloadTimeoutMin { get; }
}
}

3
Public/Src/Utilities/Configuration/Resolvers/Mutable/NuGetResolverSettings.cs
Просмотреть файл

@ -16,6 +16,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
Repositories = new Dictionary<string, string>();
Packages = new List<INugetPackage>();
DoNotEnforceDependencyVersions = false;
Configuration = new NugetConfiguration();
}
/// <nodoc />
@ -25,7 +26,7 @@ namespace BuildXL.Utilities.Configuration.Mutable
Contract.Assume(template != null);
Contract.Assume(pathRemapper != null);
Configuration = template.Configuration == null ? null : new NugetConfiguration(template.Configuration);
Configuration = new NugetConfiguration(template.Configuration);
Repositories = new Dictionary<string, string>(template.Repositories.Count);
foreach (var kv in template.Repositories)
{

15
Public/Src/Utilities/Configuration/Resolvers/Mutable/NugetConfiguration.cs
Просмотреть файл

@ -1,9 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
namespace BuildXL.Utilities.Configuration.Mutable
{
/// <nodoc />
@ -12,21 +9,17 @@ namespace BuildXL.Utilities.Configuration.Mutable
/// <nodoc />
public NugetConfiguration()
{
CredentialProviders = new List<IArtifactLocation>();
DownloadTimeoutMin = 20;
}
/// <nodoc />
public NugetConfiguration(INugetConfiguration template)
: base(template)
{
CredentialProviders = new List<IArtifactLocation>(template.CredentialProviders);
DownloadTimeoutMin = template.DownloadTimeoutMin;
}
/// <nodoc />
[SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly")]
public List<IArtifactLocation> CredentialProviders { get; set; }
/// <inheritdoc />
IReadOnlyList<IArtifactLocation> INugetConfiguration.CredentialProviders => CredentialProviders;
/// <inheritdoc/>
public int? DownloadTimeoutMin { get; set; }
}
}

2
Public/Src/Utilities/Native/BuildXL.Native.dsc
Просмотреть файл

@ -51,7 +51,7 @@ namespace Native {
export const nativeLinux = [
...addIfLazy(qualifier.targetRuntime === "linux-x64", () =>
[
...globR(d`${importFrom("runtime.linux-x64.BuildXL").Contents.all.root}/runtimes/linux-x64/native/${qualifier.configuration}`, "*.so")
...importFrom("runtime.linux-x64.BuildXL").Contents.all.ensureContents({subFolder: r`runtimes/linux-x64/native/${qualifier.configuration}`}).getContent()
]),
];

93
Public/Src/Utilities/UnitTests/Utilities/ZeroLeftPaddedStreamTests.cs Normal file
Просмотреть файл

@ -0,0 +1,93 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System.IO;
using BuildXL.Utilities;
using Test.BuildXL.TestUtilities.Xunit;
using Xunit;
using Xunit.Abstractions;
namespace Test.BuildXL.Utilities
{
public class ZeroLeftPaddedStreamTests : XunitBuildXLTest
{
public ZeroLeftPaddedStreamTests(ITestOutputHelper output)
: base(output) { }
[Fact]
public void TestBasicFunctionality()
{
var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 });
var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10);
// Total length should be 10, even if the underlying stream is smaller
XAssert.AreEqual(10, paddedStream.Length);
// Check Position works
paddedStream.Position = 9;
XAssert.AreEqual(9, paddedStream.Position);
// Read the whole stream. We should get zeros in the first 5 positions
paddedStream.Position = 0;
XAssert.AreEqual(0, paddedStream.Position);
var result = new byte[10];
var bytesRead = paddedStream.Read(result, 0, 10);
XAssert.AreArraysEqual(new byte[] { 0, 0, 0, 0, 0, 1, 2, 3, 4, 5 }, result, true);
XAssert.AreEqual(10, bytesRead);
// Read a part of the stream offsetting the destination
System.Array.Clear(result, 0, 10);
paddedStream.Position = 4;
bytesRead = paddedStream.Read(result, 1, 3);
XAssert.AreArraysEqual(new byte[] { 0, 0, 1, 2, 0, 0, 0, 0, 0, 0 }, result, true);
XAssert.AreEqual(3, bytesRead);
}
[Fact]
public void TestSeek()
{
var underlyingStream = new MemoryStream(new byte[5] { 1, 2, 3, 4, 5 });
var paddedStream = new ZeroLeftPaddedStream(underlyingStream, 10);
var result = new byte[2];
var newPos = paddedStream.Seek(4, SeekOrigin.Begin);
XAssert.AreEqual(4, newPos);
paddedStream.Read(result, 0, 2);
XAssert.AreEqual(0, result[0]);
XAssert.AreEqual(1, result[1]);
newPos = paddedStream.Seek(7, SeekOrigin.Begin);
XAssert.AreEqual(7, newPos);
paddedStream.Read(result, 0, 2);
XAssert.AreEqual(3, result[0]);
XAssert.AreEqual(4, result[1]);
newPos = paddedStream.Seek(-1, SeekOrigin.End);
XAssert.AreEqual(9, newPos);
paddedStream.Read(result, 0, 1);
XAssert.AreEqual(5, result[0]);
newPos = paddedStream.Seek(-6, SeekOrigin.End);
XAssert.AreEqual(4, newPos);
paddedStream.Read(result, 0, 2);
XAssert.AreEqual(0, result[0]);
XAssert.AreEqual(1, result[1]);
paddedStream.Position = 6;
newPos = paddedStream.Seek(-2, SeekOrigin.Current);
XAssert.AreEqual(4, newPos);
paddedStream.Read(result, 0, 2);
XAssert.AreEqual(0, result[0]);
XAssert.AreEqual(1, result[1]);
paddedStream.Position = 4;
newPos = paddedStream.Seek(2, SeekOrigin.Current);
XAssert.AreEqual(6, newPos);
paddedStream.Read(result, 0, 2);
XAssert.AreEqual(2, result[0]);
XAssert.AreEqual(3, result[1]);
}
}
}

158
Public/Src/Utilities/Utilities/ZeroLeftPaddedStream.cs Normal file
Просмотреть файл

@ -0,0 +1,158 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Diagnostics.ContractsLight;
using System.IO;
namespace BuildXL.Utilities
{
/// <summary>
/// Wraps a seekable stream to expose it with a configurable minimum size, left padding with zeros if needed.
/// </summary>
/// <remarks>
/// No extra memory is taken by the padded portion of the stream
/// If the underlying stream is modified, then the behavior of this class is non-deterministic
/// </remarks>
public class ZeroLeftPaddedStream : Stream
{
private readonly Stream m_stream;
private long m_currentPos = 0;
private readonly long m_gap;
/// <nodoc/>
public ZeroLeftPaddedStream(Stream stream, long minimumLength = 0)
{
Contract.RequiresNotNull(stream);
Contract.Requires(stream.CanSeek);
m_stream = stream;
MinimumLength = minimumLength;
m_gap = Math.Max(0, MinimumLength - m_stream.Length);
}
/// <summary>
/// The miminum legth the stream has
/// </summary>
public long MinimumLength { set; get; }
/// <inheritdoc/>
public override bool CanRead => m_stream.CanRead;
/// <inheritdoc/>
public override bool CanSeek => m_stream.CanSeek;
/// <summary>
/// Always false
/// </summary>
public override bool CanWrite => false;
/// <inheritdoc/>
public override long Length => m_stream.Length + m_gap;
/// <inheritdoc/>
public override long Position {
get => m_currentPos;
set
{
m_stream.Position = Math.Max(0, value - m_gap);
m_currentPos = Math.Min(value, Length);
}
}
/// <inheritdoc/>
public override void Flush()
{
m_stream.Flush();
}
/// <inheritdoc/>
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = 0;
if (Position < m_gap)
{
// Produce the padded area of the stream with 0-bytes
long maxPaddedCount = m_gap - Position;
int paddedCount = Math.Min(count, maxPaddedCount > int.MaxValue ? int.MaxValue : (int) maxPaddedCount);
if (paddedCount > 0)
{
Array.Clear(buffer, offset, paddedCount);
bytesRead = paddedCount;
}
// Read the reminder, if any, from the underlying stream
if (paddedCount < count)
{
bytesRead += m_stream.Read(buffer, offset + paddedCount, count - paddedCount);
}
}
else
{
// The position is beyond the padded area, just do a regular read on the underlying stream
bytesRead = m_stream.Read(buffer, offset, count);
}
Position += count;
return bytesRead;
}
/// <inheritdoc/>
public override long Seek(long offset, SeekOrigin origin)
{
// Compute the origin of the seek in terms of the starting index on the stream
long originIndex = origin switch
{
SeekOrigin.Begin => 0,
SeekOrigin.Current => Position,
SeekOrigin.End => Length,
_ => throw new ArgumentOutOfRangeException(nameof(origin), $"Not expected value {origin}")
};
// This is the final index of the stream
long finalIndex = Math.Min(Length, originIndex + offset);
if (finalIndex < 0)
{
throw new ArgumentException("Attempt to seek before the beginning of the stream");
}
if (finalIndex >= m_gap)
{
m_stream.Seek(finalIndex - m_gap, SeekOrigin.Begin);
}
else
{
m_stream.Seek(0, SeekOrigin.Begin);
}
Position = finalIndex;
return Position;
}
/// <summary>
/// Always throws
/// </summary>
/// <remarks>
/// Shouldn't be called since this stream is non-writable
/// </remarks>
public override void SetLength(long value)
{
throw new NotImplementedException();
}
/// <summary>
/// Always throws
/// </summary>
/// <remarks>
/// Shouldn't be called since this stream is non-writable
/// </remarks>
public override void Write(byte[] buffer, int offset, int count)
{
throw new NotImplementedException();
}
}
}

37
Public/Src/Utilities/VstsAuthentication/AuthenticationFailure.cs Normal file
Просмотреть файл

@ -0,0 +1,37 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
namespace BuildXL.Utilities.VstsAuthentication
{
/// <summary>
/// An authentication related failure when dealing with authenticated NuGet feeds
/// </summary>
public class AuthenticationFailure : Failure
{
private readonly string m_failure;
/// <nodoc/>
public AuthenticationFailure(string failure)
{
m_failure = failure;
}
/// <inheritdoc/>
public override BuildXLException CreateException()
{
return new BuildXLException(m_failure);
}
/// <inheritdoc/>
public override string Describe()
{
return m_failure;
}
/// <inheritdoc/>
public override BuildXLException Throw()
{
throw CreateException();
}
}
}

25
Public/Src/Utilities/VstsAuthentication/BuildXL.Utilties.VstsAuthentication.dsc Normal file
Просмотреть файл

@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
namespace VstsAuthentication {
@@public
export const dll = BuildXLSdk.library({
assemblyName: "BuildXL.Utilities.VstsAuthentication",
sources: globR(d`.`, "*.cs"),
references: [
$.dll,
Native.dll,
importFrom("Newtonsoft.Json").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Versioning").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Protocol").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Configuration").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Common").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Frameworks").withQualifier({targetFramework: "netstandard2.0"}).pkg,
importFrom("NuGet.Packaging").withQualifier({targetFramework: "netstandard2.0"}).pkg,
...addIfLazy(BuildXLSdk.isFullFramework, () => [
NetFx.System.Net.Http.dll,
NetFx.Netstandard.dll,
]),
],
});
}

334
Public/Src/Utilities/VstsAuthentication/VSTSAuthenticationHelper.cs Normal file
Просмотреть файл

@ -0,0 +1,334 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics.ContractsLight;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NuGet.Configuration;
using NuGet.Protocol;
using NuGet.Protocol.Core.Types;
using System.Net.Http;
using BuildXL.Native.IO;
using System.Text;
namespace BuildXL.Utilities.VstsAuthentication
{
/// <summary>
/// Helper to get authentication credentials against VSTS connections
/// </summary>
public static class VSTSAuthenticationHelper
{
// Constant value to target Azure DevOps.
private const string Resource = "499b84ac-1321-427f-aa17-267ca6975798";
// Visual Studio IDE client ID originally provisioned by Azure Tools.
private const string Client = "872cd9fa-d31f-45e0-9eab-6e460a02d1f1";
// Microsoft tenant
private const string Tenant = "microsoft.com";
// Microsoft authority
private const string Authority = $"https://login.windows.net/microsoft.com";
/// <summary>
/// Creates a collection of <see cref="SourceRepository"/> given a collection of URIs
/// </summary>
/// <remarks>
/// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable
/// </remarks>
public static Task<Possible<IReadOnlyCollection<SourceRepository>>> TryCreateSourceRepositories(
IEnumerable<(string repositoryName, Uri repositoryUri)> repositories,
CancellationToken cancellationToken,
StringBuilder logger)
{
return TryCreateSourceRepositories(repositories, TryDiscoverCredentialProvider, cancellationToken, logger);
}
/// <summary>
/// Creates a collection of <see cref="SourceRepository"/> given a collection of URIs
/// </summary>
/// <remarks>
/// For the repositories identified as VSTS ones, credential provider authentication is performed
/// </remarks>
public static async Task<Possible<IReadOnlyCollection<SourceRepository>>> TryCreateSourceRepositories(
IEnumerable<(string repositoryName, Uri repositoryUri)> repositories,
Func<Possible<string>> discoverCredentialProvider,
CancellationToken cancellationToken,
StringBuilder logger)
{
Contract.RequiresNotNull(repositories);
var sourceRepositories = new List<SourceRepository>();
Lazy<Possible<string>> credentialProviderPath = new Lazy<Possible<string>>(discoverCredentialProvider);
// For each repository, authenticate if we are dealing with a VSTS feed and retrieve the base addresses
foreach (var repository in repositories)
{
Uri uri = repository.repositoryUri;
string uriAsString = uri.AbsoluteUri;
var packageSource = new PackageSource(uriAsString, repository.repositoryName);
logger.AppendLine($"Analyzing {uri.AbsoluteUri}");
// For now we only support authentication against VSTS feeds. Feeds outside of that will attempt to connect unauthenticated
if (IsVSTSPackageSecureURI(uri) && await IsAuthenticationRequiredAsync(uri, cancellationToken, logger))
{
logger.AppendLine($"Authentication required for {uri.AbsoluteUri}. Trying to acquire credentials.");
if (!credentialProviderPath.Value.Succeeded)
{
return credentialProviderPath.Value.Failure;
}
if (await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath.Value.Result, cancellationToken) is var maybeCredentials)
{
var maybePackageSourceCredential = maybeCredentials
.Then(credentials => new PackageSourceCredential(uriAsString, credentials.username, credentials.pat, true, string.Empty));
if (maybePackageSourceCredential.Succeeded)
{
logger.AppendLine($"Authentication successfully acquired for {uri.AbsoluteUri}");
packageSource.Credentials = maybePackageSourceCredential.Result;
}
else
{
logger.AppendLine($"Failed to authenticate {uri.AbsoluteUri}: {maybePackageSourceCredential.Failure.DescribeIncludingInnerFailures()}");
return maybePackageSourceCredential.Failure;
}
}
}
logger.AppendLine($"Authentication not required for {uri.AbsoluteUri}");
var sourceRepository = Repository.Factory.GetCoreV3(packageSource);
sourceRepositories.Add(sourceRepository);
}
return sourceRepositories;
}
/// <summary>
/// Whether the provided Uri requires authentication
/// </summary>
public static async Task<bool> IsAuthenticationRequiredAsync(Uri uri, CancellationToken cancellationToken, StringBuilder logger)
{
logger.AppendLine($"Checking whether authentication is required for {uri.AbsoluteUri}");
try
{
var handler = new HttpClientHandler()
{
AllowAutoRedirect = false
};
using (var httpClient = new HttpClient(handler))
{
httpClient.Timeout = TimeSpan.FromSeconds(5);
int retries = 3;
while (true)
{
try
{
var httpResponse = await httpClient.GetAsync(uri, cancellationToken);
logger.AppendLine($"Response for {uri.AbsoluteUri} is: {httpResponse.StatusCode}");
return (httpResponse.StatusCode == System.Net.HttpStatusCode.Unauthorized || httpResponse.StatusCode == System.Net.HttpStatusCode.Redirect) && httpResponse.Headers.WwwAuthenticate.Any();
}
catch (TaskCanceledException)
{
logger.AppendLine($"Request for {uri.AbsoluteUri} timed out. Retries left: {retries}");
retries--;
// The service seems to be unavailable. Let the pipeline deal with the unavailability, since it will likely result in a better error message
if (retries == 0)
{
return false;
}
}
}
}
}
catch (Exception ex) when (ex is HttpRequestException || ex is AggregateException)
{
logger.AppendLine($"Exception for {uri.AbsoluteUri}: {ex.Message}");
// The service seems to be have a problem. Let's pretend it does need auth and let the rest of the pipeline deal with the unavailability, since it will likely result in a better error message
return true;
}
}
/// <summary>
/// Whether the URI is pointing to a VSTS URI
/// </summary>
public static bool IsVSTSPackageSecureURI(Uri downloadURI)
{
return downloadURI.Scheme == "https" &&
(downloadURI.Host.EndsWith("pkgs.visualstudio.com", StringComparison.OrdinalIgnoreCase) || downloadURI.Host.EndsWith("pkgs.dev.azure.com", StringComparison.OrdinalIgnoreCase));
}
/// <summary>
/// Returns an HTTP-based header from a PAT, which can be injected in an HTTP header request message
/// </summary>
public static AuthenticationHeaderValue GetAuthenticationHeaderFromPAT(string pat)
{
return new AuthenticationHeaderValue("Basic",
Convert.ToBase64String(
System.Text.ASCIIEncoding.ASCII.GetBytes(
string.Format("{0}:{1}", "", pat))));
}
/// <summary>
/// <see cref="TryGetAuthenticationCredentialsAsync(Uri, string, CancellationToken, bool)"/>
/// </summary>
/// <remarks>
/// The credential provider is discovered by searching under the paths specified in NUGET_CREDENTIALPROVIDERS_PATH environment variable
/// </remarks>
public static Task<Possible<(string username, string pat)>> TryGetAuthenticationCredentialsAsync(
Uri uri,
CancellationToken cancellationToken)
{
return TryDiscoverCredentialProvider()
.ThenAsync(async credentialProviderPath => await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken));
}
/// <summary>
/// Tries to get an authentication token using a credential provider
/// </summary>
/// <remarks>
/// The credential provider is discovered following the definition here: https://docs.microsoft.com/en-us/nuget/reference/extensibility/nuget-exe-credential-providers
/// using an environment variable.
/// Observe the link above is for nuget specifically, but the authentication here is used for VSTS feeds in general
/// </remarks>
public static Task<Possible<(string username, string pat)>> TryGetAuthenticationCredentialsAsync(
Uri uri,
string credentialProviderPath,
CancellationToken cancellationToken,
bool forceJson = false)
{
// Call the provider with the requested URI in non-interactive mode.
// Some credential providers support specifying JSON as the output format. We first try without this option (some other providers will fail if specified)
// but if the provider succeeds but we can't recognize the output as a JSON one, we try again with this option set
var processInfo = new ProcessStartInfo
{
FileName = credentialProviderPath,
Arguments = $"-Uri {uri.AbsoluteUri} -NonInteractive{(forceJson? " -F JSON" : string.Empty)}",
RedirectStandardOutput = true,
CreateNoWindow = true,
};
return RunCredentialProvider(uri, credentialProviderPath, processInfo, forceJson, cancellationToken);
}
private static async Task<Possible<(string username, string pat)>> RunCredentialProvider(
Uri uri,
string credentialProviderPath,
ProcessStartInfo processInfo,
bool jsonWasForced,
CancellationToken cancellationToken)
{
var failureCommon = $"Unable to authenticate using a credential provider for '{uri}':";
using (var process = Process.Start(processInfo))
{
try
{
#pragma warning disable AsyncFixer02 // WaitForExitAsync should be used instead
process?.WaitForExit();
#pragma warning restore AsyncFixer02
}
catch (Exception e) when (e is SystemException || e is Win32Exception)
{
return new AuthenticationFailure($"{failureCommon} Credential provider execution '{credentialProviderPath}' failed. Error: {e.Message}");
}
if (process == null)
{
// The process was not started
return new AuthenticationFailure($"{failureCommon} Could not start credential provider process '{credentialProviderPath}'.");
}
// Check whether the authorization succeeded
if (process.ExitCode == 0)
{
try
{
// The response should be a well-formed JSON with a 'password' entry representing the PAT
var response = (JObject)await JToken.ReadFromAsync(new JsonTextReader(process.StandardOutput), cancellationToken);
var pat = response.GetValue("Password");
if (pat == null)
{
return new AuthenticationFailure($"{failureCommon} Could not find a 'password' entry in the JSON response of '{credentialProviderPath}'.");
}
// We got a PAT back. Create an authentication header with a base64 encoding of the retrieved PAT
return (response.GetValue("Username")?.ToString(), pat.ToString());
}
catch (JsonReaderException)
{
// The JSON is malformed. If we were already forcing JSON, then that's an error. Otherwise, we try again but now forcing JSON
if (jsonWasForced)
{
return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' response is not a well-formed JSON.");
}
return await TryGetAuthenticationCredentialsAsync(uri, credentialProviderPath, cancellationToken, forceJson: true);
}
}
return new AuthenticationFailure($"{failureCommon} The credential provider '{credentialProviderPath}' returned a non-succesful exit code: {process.ExitCode}.");
}
}
private static Possible<string> TryDiscoverCredentialProvider()
{
var credentialProviderPaths = Environment.GetEnvironmentVariable("NUGET_CREDENTIALPROVIDERS_PATH");
if (credentialProviderPaths == null)
{
return new AuthenticationFailure("Unable to authenticate using a credential provider: NUGET_CREDENTIALPROVIDERS_PATH is not set.");
}
// Here we do something slightly simpler than what NuGet does and just look for the first credential
// provider we can find
string credentialProviderPath = null;
foreach (string path in credentialProviderPaths.Split(new[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
{
string directory = path;
// In some cases the entry can point directly to the credential .exe. Let's treat that uniformly and enumerate its parent directory
if (FileUtilities.TryProbePathExistence(path, followSymlink: false) is var maybeExistence &&
maybeExistence.Succeeded &&
maybeExistence.Result == PathExistence.ExistsAsFile)
{
directory = Directory.GetParent(path).FullName;
}
credentialProviderPath = Directory.EnumerateFiles(directory, "CredentialProvider*.exe", SearchOption.TopDirectoryOnly).FirstOrDefault();
if (credentialProviderPath != null)
{
break;
}
}
if (credentialProviderPath == null)
{
return new AuthenticationFailure($"Unable to authenticate using a credential provider: Credential provider was not found under '{credentialProviderPaths}'.");
}
return credentialProviderPath;
}
}
}

1
Shared/Scripts/Init.cmd
Просмотреть файл

@ -114,6 +114,7 @@ REM *********************************
REM We'll conditionally set the credential provider if not set on the machine.
REM If not set we will set it to the local one in the enlistment but iwth the b-drive substitution
if NOT DEFINED NUGET_CREDENTIALPROVIDERS_PATH (
ECHO NUGET_CREDENTIALPROVIDERS_PATH not set. Setting it to %TOOLROOT%
set NUGET_CREDENTIALPROVIDERS_PATH=%TOOLROOT%
)
goto :EOF

Двоичные данные
Shared/Tools/Microsoft.IdentityModel.Clients.ActiveDirectory.dll Normal file
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
Shared/Tools/Newtonsoft.Json.dll Normal file
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
Shared/Tools/NuGet.Common.dll Normal file
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
Shared/Tools/NuGet.Protocol.dll Normal file
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
Shared/Tools/PowerArgs.dll Normal file
Просмотреть файл

Двоичный файл не отображается.

3
Shared/Tools/readme.txt Normal file
Просмотреть файл

@ -0,0 +1,3 @@
These files are used by:
- bxl.ps1 in order to retrieve the latest bxl.exe from the corresponding feed and bootstrap the build with the latest LKG. Nuget.exe and the checked-in credential provider are used to authenticate against the VSTS feed.
- bxl.ps1 script also sets NUGET_CREDENTIALPROVIDERS_PATH environment variable to point to this folder (if not previously set). That implies that the Nuget resolver, if configured, will also use this credential provider to authenticate.

42
cg/nuget/cgmanifest.json
Просмотреть файл

@ -2769,7 +2769,25 @@
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.CommandLine",
"Version": "4.7.1"
"Version": "5.11.0"
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Common",
"Version": "5.11.0"
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Configuration",
"Version": "5.11.0"
}
}
},
@ -2778,7 +2796,25 @@
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Frameworks",
"Version": "5.0.0"
"Version": "5.11.0"
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Packaging",
"Version": "5.11.0"
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Protocol",
"Version": "5.11.0"
}
}
},
@ -2787,7 +2823,7 @@
"Type": "NuGet",
"NuGet": {
"Name": "NuGet.Versioning",
"Version": "4.6.0"
"Version": "5.11.0"
}
}
},

20
config.dsc
Просмотреть файл

@ -38,11 +38,10 @@ config({
]
},
{
// The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH.
kind: "Nuget",
// This configuration pins people to a specific version of nuget
// The credential provider should be set by defining the env variable NUGET_CREDENTIALPROVIDERS_PATH. TODO: It can be alternatively pinned here,
// but when it fails to download (e.g. from a share) the build is aborted. Consider making the failure non-blocking.
// TODO: remove once the new LKG gets deployed, nuget.exe is not used anymore
configuration: {
toolUrl: "https://dist.nuget.org/win-x86-commandline/v4.9.4/NuGet.exe",
hash: "VSO0:17E8C8C0CDCCA3A6D1EE49836847148C4623ACEA5E6E36E10B691DA7FDC4C39200"
@ -60,10 +59,7 @@ config({
}
: {
"buildxl-selfhost" : "https://pkgs.dev.azure.com/ms/BuildXL/_packaging/BuildXL.Selfhost/nuget/v3/index.json",
"nuget.org" : "http://api.nuget.org/v3/index.json",
"roslyn-tools" : "https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json",
"msbuild" : "https://dotnet.myget.org/F/msbuild/api/v3/index.json",
"dotnet-core" : "https://dotnet.myget.org/F/dotnet-core/api/v3/index.json",
"nuget.org" : "https://api.nuget.org/v3/index.json",
"dotnet-arcade" : "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json",
},
@ -164,9 +160,13 @@ config({
{ id: "System.Threading.Tasks.Dataflow", version: "4.9.0" },
// Nuget
{ id: "NuGet.CommandLine", version: "4.7.1" },
{ id: "NuGet.Versioning", version: "4.6.0" }, // Can't use the latest becuase nuget extracts to folder with metadata which we don't support yet.
{ id: "NuGet.Frameworks", version: "5.0.0"}, // needed for qtest on .net core
{ id: "NuGet.Packaging", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData", "System.Security.Cryptography.Pkcs"] },
{ id: "NuGet.Configuration", version: "5.11.0", dependentPackageIdsToSkip: ["System.Security.Cryptography.ProtectedData"] },
{ id: "NuGet.Common", version: "5.11.0" },
{ id: "NuGet.Protocol", version: "5.11.0" },
{ id: "NuGet.Versioning", version: "5.11.0" },
{ id: "NuGet.CommandLine", version: "5.11.0" },
{ id: "NuGet.Frameworks", version: "5.11.0"}, // needed for qtest on .net core
// ProjFS (virtual file system)
{ id: "Microsoft.Windows.ProjFS", version: "1.2.19351.1" },