Add experimental NuGet detector for framework and dev dependencies (#1285)

* Add experimental NuGet detector for framework and dev dependencies

This PR does 3 things.

1. Adds `TargetFramework` to NuGet package references.  This can be useful when querying component data to understand if components are used in a place where a vulnerability applies.
2. Adds _framework package_ handling.  The .NET SDK will do [conflict resolution](https://github.com/dotnet/sdk/tree/main/src/Tasks/Common/ConflictResolution) and drop assets from packages that overlap with the framework.  NuGet is planning to do the same https://github.com/NuGet/Home/issues/7344 but until then, it's beneficial to have component detection duplicate some of this logic.  When a package is identified as overlapping with the framework we'll treat it as a Development Dependency so that it might be auto-dismissed.
  - .NETFramework projects do not get this - .NETFramework does not participate in conflict resolution by default.  Even when enabled framework assemblies can be bypassed using bindingRedirects, or avoiding references to them.  Due this fragility it's not safe to apply framework package rules to .NETFramework.
  - packages.config usage is also excluded since it precludes SDK conflict resolution (and is also only used on .NETFramework projects).
3. Recognizes `ExcludeAssets="Runtime"` usage as a Development Dependencies, also any packages which don't contribute to "runtime" will be developement dependencies.

I reused _Development Dependency_ rather than plumbing a new concept.
I only mapped data for the `Microsoft.NETCore.App` - the default shared framework.  We could consider doing the same for `Microsoft.ASPNETCore.App` and `Microsoft.WindowsDesktop.App` but we'd need to plumb the reference information out of the assets file - currently that's not read and I'm not aware of a supported NuGet API for reading it (though it is present under `project/frameworks/<framework>/frameworkReferences/<name>`
.NET Core 1.x has no data since it was packages itself.  I have a fallback for future frameworks to read the data from the targeting packs.

* Address feedback
This commit is contained in:
Eric StJohn 2024-10-31 11:52:23 -07:00 коммит произвёл GitHub
Родитель a0e15204f2
Коммит c2546faf1e
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
21 изменённых файлов: 1304 добавлений и 3 удалений

Просмотреть файл

@ -1,5 +1,6 @@
namespace Microsoft.ComponentDetection.Contracts.TypedComponent;
using System.Collections.Generic;
using PackageUrl;
public class NuGetComponent : TypedComponent
@ -22,6 +23,8 @@ public class NuGetComponent : TypedComponent
public string[] Authors { get; set; }
public ISet<string> TargetFrameworks { get; set; } = new HashSet<string>();
public override ComponentType Type => ComponentType.NuGet;
public override string Id => $"{this.Name} {this.Version} - {this.Type}";

Просмотреть файл

@ -0,0 +1,122 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using global::NuGet.Frameworks;
using global::NuGet.Versioning;
/// <summary>
/// Represents a set of packages that are provided by a specific framework.
/// At the moment this only represents the packages that are provided by the Microsoft.NETCore.App framework.
/// We could extend this to represent the packages provided by other frameworks like Microsoft.AspNetCore.App and Microsoft.WindowsDesktop.App.
/// </summary>
internal sealed partial class FrameworkPackages : IEnumerable<KeyValuePair<string, NuGetVersion>>, IEnumerable
{
private static readonly Dictionary<NuGetFramework, FrameworkPackages> FrameworkPackagesByFramework = [];
static FrameworkPackages()
{
AddPackages(NETStandard20.Instance);
AddPackages(NETStandard21.Instance);
AddPackages(NETCoreApp20.Instance);
AddPackages(NETCoreApp21.Instance);
AddPackages(NETCoreApp22.Instance);
AddPackages(NETCoreApp30.Instance);
AddPackages(NETCoreApp31.Instance);
AddPackages(NETCoreApp50.Instance);
AddPackages(NETCoreApp60.Instance);
AddPackages(NETCoreApp70.Instance);
AddPackages(NETCoreApp80.Instance);
AddPackages(NETCoreApp90.Instance);
static void AddPackages(FrameworkPackages packages) => FrameworkPackagesByFramework[packages.Framework] = packages;
}
public FrameworkPackages(NuGetFramework framework) => this.Framework = framework;
public FrameworkPackages(NuGetFramework framework, FrameworkPackages frameworkPackages)
: this(framework) => this.Packages = new(frameworkPackages.Packages);
public NuGetFramework Framework { get; }
public Dictionary<string, NuGetVersion> Packages { get; } = new Dictionary<string, NuGetVersion>(StringComparer.OrdinalIgnoreCase);
public static FrameworkPackages GetFrameworkPackages(NuGetFramework framework)
{
if (FrameworkPackagesByFramework.TryGetValue(framework, out var frameworkPackages))
{
return frameworkPackages;
}
// if we didn't predefine the package overrides, load them from the targeting pack
// we might just leave this out since in future frameworks we'll have this functionality built into NuGet.
var frameworkPackagesFromPack = LoadFrameworkPackagesFromPack(framework);
return FrameworkPackagesByFramework[framework] = frameworkPackagesFromPack ?? new FrameworkPackages(framework);
}
private static FrameworkPackages LoadFrameworkPackagesFromPack(NuGetFramework framework)
{
if (framework is null || framework.Framework != FrameworkConstants.FrameworkIdentifiers.NetCoreApp)
{
return null;
}
// packs location : %ProgramFiles%\dotnet\packs
var packsFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "dotnet", "packs", "Microsoft.NETCore.App.Ref");
if (!Directory.Exists(packsFolder))
{
return null;
}
var packVersionPattern = $"{framework.Version.Major}.{framework.Version.Minor}.*";
var packDirectories = Directory.GetDirectories(packsFolder, packVersionPattern);
var packageOverridesFile = packDirectories
.Select(d => (Overrides: Path.Combine(d, "data", "PackageOverrides.txt"), Version: ParseVersion(Path.GetFileName(d))))
.Where(d => File.Exists(d.Overrides))
.OrderByDescending(d => d.Version)
.FirstOrDefault().Overrides;
if (packageOverridesFile == null)
{
// we should also try to grab them from the user's package folder - they'll be in one location or the other.
return null;
}
// Adapted from https://github.com/dotnet/sdk/blob/c3a8f72c3a5491c693ff8e49e7406136a12c3040/src/Tasks/Common/ConflictResolution/PackageOverride.cs#L52-L68
var frameworkPackages = new FrameworkPackages(framework);
var packageOverrides = File.ReadAllLines(packageOverridesFile);
foreach (var packageOverride in packageOverrides)
{
var packageOverrideParts = packageOverride.Trim().Split('|');
if (packageOverrideParts.Length == 2)
{
var packageId = packageOverrideParts[0];
var packageVersion = ParseVersion(packageOverrideParts[1]);
frameworkPackages.Packages[packageId] = packageVersion;
}
}
return frameworkPackages;
static NuGetVersion ParseVersion(string versionString) => NuGetVersion.TryParse(versionString, out var version) ? version : null;
}
private void Add(string id, string version)
{
// intentionally redirect to indexer to allow for overwrite
this.Packages[id] = NuGetVersion.Parse(version);
}
public bool IsAFrameworkComponent(string id, NuGetVersion version) => this.Packages.TryGetValue(id, out var frameworkPackageVersion) && frameworkPackageVersion >= version;
IEnumerator<KeyValuePair<string, NuGetVersion>> IEnumerable<KeyValuePair<string, NuGetVersion>>.GetEnumerator() => this.Packages.GetEnumerator();
IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException();
}

Просмотреть файл

@ -0,0 +1,81 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for net5.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp50
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("net5.0"), NETCoreApp31.Instance)
{
{ "Microsoft.CSharp", "4.7.0" },
{ "runtime.debian.8-x64.runtime.native.System", "4.3.0" },
{ "runtime.debian.8-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.debian.8-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.debian.8-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.debian.8-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.IO.Compression", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Net.Http", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Net.Security", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography", "4.3.0" },
{ "System.Buffers", "4.5.1" },
{ "System.Collections.Immutable", "5.0.0" },
{ "System.ComponentModel.Annotations", "5.0.0" },
{ "System.Diagnostics.DiagnosticSource", "5.0.0" },
{ "System.Formats.Asn1", "5.0.0" },
{ "System.Net.Http.Json", "5.0.0" },
{ "System.Reflection.DispatchProxy", "4.7.1" },
{ "System.Reflection.Metadata", "5.0.0" },
{ "System.Runtime.CompilerServices.Unsafe", "5.0.0" },
{ "System.Text.Encoding.CodePages", "5.0.0" },
{ "System.Text.Encodings.Web", "5.0.0" },
{ "System.Text.Json", "5.0.0" },
{ "System.Threading.Channels", "5.0.0" },
{ "System.Threading.Tasks.Dataflow", "5.0.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,34 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for net6.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp60
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("net6.0"), NETCoreApp50.Instance)
{
{ "Microsoft.Win32.Registry", "5.0.0" },
{ "System.Collections.Immutable", "6.0.0" },
{ "System.Diagnostics.DiagnosticSource", "6.0.1" },
{ "System.Formats.Asn1", "6.0.1" },
{ "System.IO.FileSystem.AccessControl", "5.0.0" },
{ "System.IO.Pipes.AccessControl", "5.0.0" },
{ "System.Net.Http.Json", "6.0.1" },
{ "System.Reflection.Metadata", "6.0.1" },
{ "System.Runtime.CompilerServices.Unsafe", "6.0.0" },
{ "System.Security.AccessControl", "6.0.1" },
{ "System.Security.Cryptography.Cng", "5.0.0" },
{ "System.Security.Cryptography.OpenSsl", "5.0.0" },
{ "System.Security.Principal.Windows", "5.0.0" },
{ "System.Text.Encoding.CodePages", "6.0.0" },
{ "System.Text.Encodings.Web", "6.0.0" },
{ "System.Text.Json", "6.0.9" },
{ "System.Threading.Channels", "6.0.0" },
{ "System.Threading.Tasks.Dataflow", "6.0.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,26 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for net7.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp70
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("net7.0"), NETCoreApp60.Instance)
{
{ "System.Collections.Immutable", "7.0.0" },
{ "System.Diagnostics.DiagnosticSource", "7.0.2" },
{ "System.Formats.Asn1", "7.0.0" },
{ "System.Net.Http.Json", "7.0.1" },
{ "System.Reflection.Metadata", "7.0.2" },
{ "System.Text.Encoding.CodePages", "7.0.0" },
{ "System.Text.Encodings.Web", "7.0.0" },
{ "System.Text.Json", "7.0.4" },
{ "System.Threading.Channels", "7.0.0" },
{ "System.Threading.Tasks.Dataflow", "7.0.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,26 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for net8.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp80
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("net8.0"), NETCoreApp70.Instance)
{
{ "System.Collections.Immutable", "8.0.0" },
{ "System.Diagnostics.DiagnosticSource", "8.0.1" },
{ "System.Formats.Asn1", "8.0.1" },
{ "System.Net.Http.Json", "8.0.0" },
{ "System.Reflection.Metadata", "8.0.0" },
{ "System.Text.Encoding.CodePages", "8.0.0" },
{ "System.Text.Encodings.Web", "8.0.0" },
{ "System.Text.Json", "8.0.4" },
{ "System.Threading.Channels", "8.0.0" },
{ "System.Threading.Tasks.Dataflow", "8.0.1" },
};
}
}

Просмотреть файл

@ -0,0 +1,98 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for net9.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp90
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("net9.0"), NETCoreApp80.Instance)
{
{ "Microsoft.VisualBasic", "10.4.0" },
{ "runtime.debian.8-x64.runtime.native.System", "4.3.1" },
{ "runtime.debian.8-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.debian.8-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.debian.8-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.debian.8-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.fedora.23-x64.runtime.native.System", "4.3.1" },
{ "runtime.fedora.23-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.fedora.23-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.fedora.23-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.fedora.23-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.fedora.24-x64.runtime.native.System", "4.3.1" },
{ "runtime.fedora.24-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.fedora.24-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.fedora.24-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.fedora.24-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.opensuse.13.2-x64.runtime.native.System", "4.3.1" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.opensuse.42.1-x64.runtime.native.System", "4.3.1" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.osx.10.10-x64.runtime.native.System", "4.3.1" },
{ "runtime.osx.10.10-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.osx.10.10-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.osx.10.10-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple", "4.3.1" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.rhel.7-x64.runtime.native.System", "4.3.1" },
{ "runtime.rhel.7-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.rhel.7-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.rhel.7-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.rhel.7-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System", "4.3.1" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System", "4.3.1" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System", "4.3.1" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.IO.Compression", "4.3.2" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Net.Http", "4.3.1" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Net.Security", "4.3.1" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography", "4.3.4" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.3" },
{ "System.Buffers", "5.0.0" },
{ "System.Collections.Immutable", "9.0.0" },
{ "System.Diagnostics.DiagnosticSource", "9.0.0" },
{ "System.Formats.Asn1", "9.0.0" },
{ "System.Formats.Tar", "9.0.0" },
{ "System.IO.Pipelines", "9.0.0" },
{ "System.Memory", "5.0.0" },
{ "System.Net.Http.Json", "9.0.0" },
{ "System.Numerics.Vectors", "5.0.0" },
{ "System.Private.Uri", "4.3.2" },
{ "System.Reflection.DispatchProxy", "6.0.0" },
{ "System.Reflection.Metadata", "9.0.0" },
{ "System.Runtime.CompilerServices.Unsafe", "7.0.0" },
{ "System.Text.Encoding.CodePages", "9.0.0" },
{ "System.Text.Encodings.Web", "9.0.0" },
{ "System.Text.Json", "9.0.0" },
{ "System.Threading.Channels", "9.0.0" },
{ "System.Threading.Tasks.Dataflow", "9.0.0" },
{ "System.Threading.Tasks.Extensions", "5.0.0" },
{ "System.Xml.XPath.XDocument", "5.0.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,49 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETCoreApp,Version=v2.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp20
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netcoreapp2.0"), NETStandard20.Instance)
{
{ "System.Buffers", "4.4.0" },
{ "System.Collections.Concurrent", "4.3.0" },
{ "System.Collections.Immutable", "1.4.0" },
{ "System.ComponentModel", "4.3.0" },
{ "System.ComponentModel.Annotations", "4.4.0" },
{ "System.ComponentModel.EventBasedAsync", "4.3.0" },
{ "System.Diagnostics.Contracts", "4.3.0" },
{ "System.Diagnostics.DiagnosticSource", "4.4.1" },
{ "System.Dynamic.Runtime", "4.3.0" },
{ "System.Linq.Parallel", "4.3.0" },
{ "System.Linq.Queryable", "4.3.0" },
{ "System.Net.Requests", "4.3.0" },
{ "System.Net.WebHeaderCollection", "4.3.0" },
{ "System.Numerics.Vectors", "4.4.0" },
{ "System.ObjectModel", "4.3.0" },
{ "System.Reflection.DispatchProxy", "4.4.0" },
{ "System.Reflection.Emit", "4.7.0" },
{ "System.Reflection.Emit.ILGeneration", "4.7.0" },
{ "System.Reflection.Emit.Lightweight", "4.7.0" },
{ "System.Reflection.Metadata", "1.5.0" },
{ "System.Reflection.TypeExtensions", "4.7.0" },
{ "System.Runtime.InteropServices.WindowsRuntime", "4.3.0" },
{ "System.Runtime.Loader", "4.3.0" },
{ "System.Runtime.Numerics", "4.3.0" },
{ "System.Runtime.Serialization.Json", "4.3.0" },
{ "System.Security.Principal", "4.3.0" },
{ "System.Text.RegularExpressions", "4.3.1" },
{ "System.Threading", "4.3.0" },
{ "System.Threading.Tasks.Dataflow", "4.8.0" },
{ "System.Threading.Tasks.Extensions", "4.4.0" },
{ "System.Threading.Tasks.Parallel", "4.3.0" },
{ "System.Xml.XDocument", "4.3.0" },
{ "System.Xml.XmlSerializer", "4.3.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,25 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETCoreApp,Version=v2.1.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp21
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netcoreapp2.1"), NETCoreApp20.Instance)
{
{ "System.Collections.Immutable", "1.5.0" },
{ "System.ComponentModel.Annotations", "4.4.1" },
{ "System.Diagnostics.DiagnosticSource", "4.5.0" },
{ "System.Memory", "4.5.5" },
{ "System.Reflection.DispatchProxy", "4.5.0" },
{ "System.Reflection.Metadata", "1.6.0" },
{ "System.Threading.Tasks.Dataflow", "4.9.0" },
{ "System.Threading.Tasks.Extensions", "4.5.4" },
{ "System.ValueTuple", "4.5.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,15 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETCoreApp,Version=v2.2.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp22
{
// .NETCore 2.2 was the same as .NETCore 2.1
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netcoreapp2.2"), NETCoreApp21.Instance);
}
}

Просмотреть файл

@ -0,0 +1,49 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETCoreApp,Version=v3.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp30
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netcoreapp3.0"), NETCoreApp21.Instance)
{
{ "Microsoft.CSharp", "4.4.0" },
{ "Microsoft.Win32.Registry", "4.4.0" },
{ "runtime.debian.8-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.fedora.23-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.fedora.24-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.opensuse.13.2-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.opensuse.42.1-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.Apple", "4.3.0" },
{ "runtime.osx.10.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.rhel.7-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.ubuntu.14.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.ubuntu.16.04-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "runtime.ubuntu.16.10-x64.runtime.native.System.Security.Cryptography.OpenSsl", "4.3.0" },
{ "System.Collections.Immutable", "1.6.0" },
{ "System.ComponentModel.Annotations", "4.6.0" },
{ "System.Data.DataSetExtensions", "4.5.0" },
{ "System.Diagnostics.DiagnosticSource", "4.6.0" },
{ "System.IO.FileSystem.AccessControl", "4.4.0" },
{ "System.Numerics.Vectors", "4.5.0" },
{ "System.Private.DataContractSerialization", "4.3.0" },
{ "System.Reflection.DispatchProxy", "4.6.0" },
{ "System.Reflection.Metadata", "1.7.0" },
{ "System.Runtime.CompilerServices.Unsafe", "4.6.0" },
{ "System.Security.AccessControl", "4.4.0" },
{ "System.Security.Cryptography.Cng", "4.4.0" },
{ "System.Security.Cryptography.OpenSsl", "4.4.0" },
{ "System.Security.Cryptography.Xml", "4.4.0" },
{ "System.Security.Principal.Windows", "4.4.0" },
{ "System.Text.Encoding.CodePages", "4.6.0" },
{ "System.Text.Encodings.Web", "4.6.0" },
{ "System.Text.Json", "4.6.0" },
{ "System.Threading.Channels", "4.6.0" },
{ "System.Threading.Tasks.Dataflow", "4.10.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,27 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETCoreApp,Version=v3.1.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETCoreApp31
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netcoreapp3.1"), NETCoreApp30.Instance)
{
{ "System.Collections.Immutable", "1.7.0" },
{ "System.ComponentModel.Annotations", "4.7.0" },
{ "System.Diagnostics.DiagnosticSource", "4.7.0" },
{ "System.Reflection.DispatchProxy", "4.7.0" },
{ "System.Reflection.Metadata", "1.8.0" },
{ "System.Runtime.CompilerServices.Unsafe", "4.7.1" },
{ "System.Text.Encoding.CodePages", "4.7.0" },
{ "System.Text.Encodings.Web", "4.7.0" },
{ "System.Text.Json", "4.7.0" },
{ "System.Threading.Channels", "4.7.0" },
{ "System.Threading.Tasks.Dataflow", "4.11.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,108 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETStandard,Version=v2.0.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETStandard20
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netstandard2.0"))
{
{ "Microsoft.Win32.Primitives", "4.3.0" },
{ "System.AppContext", "4.3.0" },
{ "System.Collections", "4.3.0" },
{ "System.Collections.NonGeneric", "4.3.0" },
{ "System.Collections.Specialized", "4.3.0" },
{ "System.ComponentModel", "4.0.1" },
{ "System.ComponentModel.EventBasedAsync", "4.0.11" },
{ "System.ComponentModel.Primitives", "4.3.0" },
{ "System.ComponentModel.TypeConverter", "4.3.0" },
{ "System.Console", "4.3.1" },
{ "System.Data.Common", "4.3.0" },
{ "System.Diagnostics.Contracts", "4.0.1" },
{ "System.Diagnostics.Debug", "4.3.0" },
{ "System.Diagnostics.FileVersionInfo", "4.3.0" },
{ "System.Diagnostics.Process", "4.3.0" },
{ "System.Diagnostics.StackTrace", "4.3.0" },
{ "System.Diagnostics.TextWriterTraceListener", "4.3.0" },
{ "System.Diagnostics.Tools", "4.3.0" },
{ "System.Diagnostics.TraceSource", "4.3.0" },
{ "System.Diagnostics.Tracing", "4.3.0" },
{ "System.Drawing.Primitives", "4.3.0" },
{ "System.Dynamic.Runtime", "4.0.11" },
{ "System.Globalization", "4.3.0" },
{ "System.Globalization.Calendars", "4.3.0" },
{ "System.Globalization.Extensions", "4.3.0" },
{ "System.IO", "4.3.0" },
{ "System.IO.Compression", "4.3.0" },
{ "System.IO.Compression.ZipFile", "4.3.0" },
{ "System.IO.FileSystem", "4.3.0" },
{ "System.IO.FileSystem.DriveInfo", "4.3.1" },
{ "System.IO.FileSystem.Primitives", "4.3.0" },
{ "System.IO.FileSystem.Watcher", "4.3.0" },
{ "System.IO.IsolatedStorage", "4.3.0" },
{ "System.IO.MemoryMappedFiles", "4.3.0" },
{ "System.IO.Pipes", "4.3.0" },
{ "System.IO.UnmanagedMemoryStream", "4.3.0" },
{ "System.Linq", "4.3.0" },
{ "System.Linq.Expressions", "4.3.0" },
{ "System.Linq.Parallel", "4.0.1" },
{ "System.Linq.Queryable", "4.0.1" },
{ "System.Net.Http", "4.3.4" },
{ "System.Net.NameResolution", "4.3.0" },
{ "System.Net.NetworkInformation", "4.3.0" },
{ "System.Net.Ping", "4.3.0" },
{ "System.Net.Primitives", "4.3.1" },
{ "System.Net.Requests", "4.0.11" },
{ "System.Net.Security", "4.3.2" },
{ "System.Net.Sockets", "4.3.0" },
{ "System.Net.WebHeaderCollection", "4.0.1" },
{ "System.Net.WebSockets", "4.3.0" },
{ "System.Net.WebSockets.Client", "4.3.2" },
{ "System.Reflection", "4.3.0" },
{ "System.Reflection.Extensions", "4.3.0" },
{ "System.Reflection.Primitives", "4.3.0" },
{ "System.Resources.Reader", "4.3.0" },
{ "System.Resources.ResourceManager", "4.3.0" },
{ "System.Resources.Writer", "4.3.0" },
{ "System.Runtime", "4.3.1" },
{ "System.Runtime.CompilerServices.VisualC", "4.3.0" },
{ "System.Runtime.Extensions", "4.3.1" },
{ "System.Runtime.Handles", "4.3.0" },
{ "System.Runtime.InteropServices", "4.3.0" },
{ "System.Runtime.InteropServices.RuntimeInformation", "4.3.0" },
{ "System.Runtime.Numerics", "4.0.1" },
{ "System.Runtime.Serialization.Formatters", "4.3.0" },
{ "System.Runtime.Serialization.Primitives", "4.3.0" },
{ "System.Runtime.Serialization.Xml", "4.3.0" },
{ "System.Security.Claims", "4.3.0" },
{ "System.Security.Cryptography.Algorithms", "4.3.1" },
{ "System.Security.Cryptography.Csp", "4.3.0" },
{ "System.Security.Cryptography.Encoding", "4.3.0" },
{ "System.Security.Cryptography.Primitives", "4.3.0" },
{ "System.Security.Cryptography.X509Certificates", "4.3.2" },
{ "System.Security.Principal", "4.0.1" },
{ "System.Security.SecureString", "4.3.0" },
{ "System.Text.Encoding", "4.3.0" },
{ "System.Text.Encoding.Extensions", "4.3.0" },
{ "System.Text.RegularExpressions", "4.3.0" },
{ "System.Threading", "4.0.11" },
{ "System.Threading.Overlapped", "4.3.0" },
{ "System.Threading.Tasks", "4.3.0" },
{ "System.Threading.Tasks.Parallel", "4.0.1" },
{ "System.Threading.Thread", "4.3.0" },
{ "System.Threading.ThreadPool", "4.3.0" },
{ "System.Threading.Timer", "4.3.0" },
{ "System.ValueTuple", "4.4.0" },
{ "System.Xml.ReaderWriter", "4.3.1" },
{ "System.Xml.XDocument", "4.0.11" },
{ "System.Xml.XmlDocument", "4.3.0" },
{ "System.Xml.XmlSerializer", "4.0.11" },
{ "System.Xml.XPath", "4.3.0" },
{ "System.Xml.XPath.XDocument", "4.3.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,48 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using global::NuGet.Frameworks;
/// <summary>
/// Framework packages for .NETStandard,Version=v2.1.
/// </summary>
internal partial class FrameworkPackages
{
internal static class NETStandard21
{
internal static FrameworkPackages Instance { get; } = new(NuGetFramework.Parse("netstandard2.1"), NETStandard20.Instance)
{
{ "System.Buffers", "4.5.1" },
{ "System.Collections.Concurrent", "4.3.0" },
{ "System.Collections.Immutable", "1.4.0" },
{ "System.ComponentModel", "4.3.0" },
{ "System.ComponentModel.Composition", "4.5.0" },
{ "System.ComponentModel.EventBasedAsync", "4.3.0" },
{ "System.Diagnostics.Contracts", "4.3.0" },
{ "System.Dynamic.Runtime", "4.3.0" },
{ "System.Linq.Queryable", "4.3.0" },
{ "System.Memory", "4.5.5" },
{ "System.Net.Requests", "4.3.0" },
{ "System.Net.WebHeaderCollection", "4.3.0" },
{ "System.Numerics.Vectors", "4.5.0" },
{ "System.ObjectModel", "4.3.0" },
{ "System.Private.DataContractSerialization", "4.3.0" },
{ "System.Reflection.DispatchProxy", "4.5.1" },
{ "System.Reflection.Emit", "4.7.0" },
{ "System.Reflection.Emit.ILGeneration", "4.7.0" },
{ "System.Reflection.Emit.Lightweight", "4.7.0" },
{ "System.Reflection.TypeExtensions", "4.3.0" },
{ "System.Runtime.Loader", "4.3.0" },
{ "System.Runtime.Numerics", "4.3.0" },
{ "System.Runtime.Serialization.Json", "4.3.0" },
{ "System.Security.AccessControl", "4.4.0" },
{ "System.Security.Cryptography.Xml", "4.4.0" },
{ "System.Security.Principal", "4.3.0" },
{ "System.Security.Principal.Windows", "4.4.0" },
{ "System.Threading", "4.3.0" },
{ "System.Threading.Tasks.Extensions", "4.5.4" },
{ "System.Threading.Tasks.Parallel", "4.3.0" },
{ "System.Xml.XDocument", "4.3.0" },
{ "System.Xml.XmlSerializer", "4.3.0" },
};
}
}

Просмотреть файл

@ -0,0 +1,7 @@
The .NET SDK does conflict resolution with NuGet packages to decide when to ignore the content of packages in favor of that from other sources (like other packages or the Framework itself).
Since packages are immutable - we can precompute what packages will "lose" to framework assets. These lists were calculated with a tool that downloads packages, resolve what assets apply for a given frameowrk, then compares those to the framework's reference assemblies.
The framework targeting packs have a file to ship these precomputed package versions, [PackageOverrides.txt](https://github.com/dotnet/sdk/blob/7deb36232b9c0ccd5084fced1df07920c10a5b72/src/Tasks/Microsoft.NET.Build.Tasks/ResolveTargetingPackAssets.cs#L199) -- but that file wasn't kept up to date of the course framework versions. It was meant as a "fast path" not a source of truth. In future framework versions this will need to be the source of truth since it will feed into NuGet's supplied by platform feature. The latest version of this file for net9.0 can be seen [here](https://github.com/dotnet/runtime/blob/release/9.0/src/installer/pkg/sfx/Microsoft.NETCore.App/PackageOverrides.txt) and is included in the [targeting pack package](https://www.nuget.org/packages/Microsoft.NETCore.App.Ref).
Once caclculating these we reduce them to a minimum set by allowing compatible frameworks to build upon the previous framework's data - thus reducing the total code size and memory usage of the set of framework packages.

Просмотреть файл

@ -0,0 +1,223 @@
namespace Microsoft.ComponentDetection.Detectors.NuGet;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using global::NuGet.Packaging.Core;
using global::NuGet.ProjectModel;
using global::NuGet.Versioning;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.Internal;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.Extensions.Logging;
public class NuGetPackageReferenceFrameworkAwareDetector : FileComponentDetector, IExperimentalDetector
{
public const string OmittedFrameworkComponentsTelemetryKey = "OmittedFrameworkComponents";
public const string ProjectDependencyType = "project";
private readonly IFileUtilityService fileUtilityService;
public NuGetPackageReferenceFrameworkAwareDetector(
IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
IObservableDirectoryWalkerFactory walkerFactory,
IFileUtilityService fileUtilityService,
ILogger<NuGetPackageReferenceFrameworkAwareDetector> logger)
{
this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory;
this.Scanner = walkerFactory;
this.fileUtilityService = fileUtilityService;
this.Logger = logger;
}
public override string Id { get; } = "NuGetPackageReferenceFrameworkAware";
public override IEnumerable<string> Categories => [Enum.GetName(typeof(DetectorClass), DetectorClass.NuGet)];
public override IList<string> SearchPatterns { get; } = ["project.assets.json"];
public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = [ComponentType.NuGet];
public override int Version { get; } = 1;
protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
{
try
{
var lockFile = new LockFileFormat().Read(processRequest.ComponentStream.Stream, processRequest.ComponentStream.Location);
this.RecordLockfileVersion(lockFile.Version);
if (lockFile.PackageSpec == null)
{
throw new FormatException("Lockfile did not contain a PackageSpec");
}
var explicitReferencedDependencies = this.GetTopLevelLibraries(lockFile)
.Select(x => this.GetLibraryComponentWithDependencyLookup(lockFile.Libraries, x.Name, x.Version, x.VersionRange))
.ToList();
var explicitlyReferencedComponentIds =
explicitReferencedDependencies
.Select(x => new NuGetComponent(x.Name, x.Version.ToNormalizedString()).Id)
.ToHashSet();
// Since we report projects as the location, we ignore the passed in single file recorder.
var singleFileComponentRecorder = this.ComponentRecorder.CreateSingleFileComponentRecorder(lockFile.PackageSpec.RestoreMetadata.ProjectPath);
foreach (var target in lockFile.Targets)
{
var frameworkPackages = FrameworkPackages.GetFrameworkPackages(target.TargetFramework);
// This call to GetTargetLibrary is not guarded, because if this can't be resolved then something is fundamentally broken (e.g. an explicit dependency reference not being in the list of libraries)
// issue: we treat top level dependencies for all targets as top level for each target, but some may not be top level for other targets, or may not even be present for other targets.
foreach (var library in explicitReferencedDependencies.Select(x => target.GetTargetLibrary(x.Name)).Where(x => x != null))
{
this.NavigateAndRegister(target, explicitlyReferencedComponentIds, singleFileComponentRecorder, library, null, frameworkPackages);
}
}
}
catch (Exception e)
{
// If something went wrong, just ignore the package
this.Logger.LogError(e, "Failed to process NuGet lockfile {NuGetLockFile}", processRequest.ComponentStream.Location);
}
return Task.CompletedTask;
}
private void NavigateAndRegister(
LockFileTarget target,
HashSet<string> explicitlyReferencedComponentIds,
ISingleFileComponentRecorder singleFileComponentRecorder,
LockFileTargetLibrary library,
string parentComponentId,
FrameworkPackages frameworkPackages,
HashSet<string> visited = null)
{
if (library.Type == ProjectDependencyType)
{
return;
}
var isFrameworkComponent = frameworkPackages?.IsAFrameworkComponent(library.Name, library.Version) ?? false;
var isDevelopmentDependency = IsADevelopmentDependency(library);
visited ??= [];
var libraryComponent = new DetectedComponent(new NuGetComponent(library.Name, library.Version.ToNormalizedString()));
singleFileComponentRecorder.RegisterUsage(libraryComponent, explicitlyReferencedComponentIds.Contains(libraryComponent.Component.Id), parentComponentId, isDevelopmentDependency: isFrameworkComponent || isDevelopmentDependency);
// get the actual component in case it already exists
libraryComponent = singleFileComponentRecorder.GetComponent(libraryComponent.Component.Id);
// Add framework information to the actual component
if (target.TargetFramework is not null)
{
((NuGetComponent)libraryComponent.Component).TargetFrameworks.Add(target.TargetFramework.GetShortFolderName());
}
foreach (var dependency in library.Dependencies)
{
if (visited.Contains(dependency.Id))
{
continue;
}
var targetLibrary = target.GetTargetLibrary(dependency.Id);
if (targetLibrary == null)
{
// We have to exclude this case -- it looks like a bug in project.assets.json, but there are project.assets.json files that don't have a dependency library in the libraries set.
}
else
{
visited.Add(dependency.Id);
this.NavigateAndRegister(target, explicitlyReferencedComponentIds, singleFileComponentRecorder, targetLibrary, libraryComponent.Component.Id, frameworkPackages, visited);
}
}
// a placeholder item is an empty file that doesn't exist with name _._ meant to indicate an empty folder in a nuget package, but also used by NuGet when a package's assets are excluded.
bool IsAPlaceholderItem(LockFileItem item) => Path.GetFileName(item.Path).Equals(PackagingCoreConstants.EmptyFolder, StringComparison.OrdinalIgnoreCase);
// A library is development dependency if all of the runtime assemblies and runtime targets are placeholders or empty (All returns true for empty).
bool IsADevelopmentDependency(LockFileTargetLibrary library) => library.RuntimeAssemblies.Concat(library.RuntimeTargets).All(IsAPlaceholderItem);
}
private List<(string Name, Version Version, VersionRange VersionRange)> GetTopLevelLibraries(LockFile lockFile)
{
// First, populate libraries from the TargetFrameworks section -- This is the base level authoritative list of nuget packages a project has dependencies on.
var toBeFilled = new List<(string Name, Version Version, VersionRange VersionRange)>();
foreach (var framework in lockFile.PackageSpec.TargetFrameworks)
{
foreach (var dependency in framework.Dependencies)
{
toBeFilled.Add((dependency.Name, Version: null, dependency.LibraryRange.VersionRange));
}
}
// Next, we need to resolve project references -- This is a little funky, because project references are only stored via path in
// project.assets.json, so we first build a list of all paths and then compare what is top level to them to resolve their
// associated library.
var projectDirectory = Path.GetDirectoryName(lockFile.PackageSpec.RestoreMetadata.ProjectPath);
var librariesWithAbsolutePath =
lockFile.Libraries.Where(x => x.Type == ProjectDependencyType)
.Select(x => (library: x, absoluteProjectPath: Path.GetFullPath(Path.Combine(projectDirectory, x.Path))))
.ToDictionary(x => x.absoluteProjectPath, x => x.library);
foreach (var restoreMetadataTargetFramework in lockFile.PackageSpec.RestoreMetadata.TargetFrameworks)
{
foreach (var projectReference in restoreMetadataTargetFramework.ProjectReferences)
{
if (librariesWithAbsolutePath.TryGetValue(Path.GetFullPath(projectReference.ProjectPath), out var library))
{
toBeFilled.Add((library.Name, library.Version.Version, null));
}
}
}
return toBeFilled;
}
// Looks up a library in project.assets.json given a version (preferred) or version range (have to in some cases due to how project.assets.json stores things)
private LockFileLibrary GetLibraryComponentWithDependencyLookup(IList<LockFileLibrary> libraries, string dependencyId, Version version, VersionRange versionRange)
{
if ((version == null && versionRange == null) || (version != null && versionRange != null))
{
throw new ArgumentException($"Either {nameof(version)} or {nameof(versionRange)} must be specified, but not both.");
}
var matchingLibraryNames = libraries.Where(x => string.Equals(x.Name, dependencyId, StringComparison.OrdinalIgnoreCase)).ToList();
if (matchingLibraryNames.Count == 0)
{
throw new InvalidOperationException("Project.assets.json is malformed, no library could be found matching: " + dependencyId);
}
LockFileLibrary matchingLibrary;
if (version != null)
{
// .Version.Version ensures we get to a nuget normalized 4 part version
matchingLibrary = matchingLibraryNames.FirstOrDefault(x => x.Version.Version.Equals(version));
}
else
{
matchingLibrary = matchingLibraryNames.FirstOrDefault(x => versionRange.Satisfies(x.Version));
}
if (matchingLibrary == null)
{
matchingLibrary = matchingLibraryNames.First();
var versionString = versionRange != null ? versionRange.ToNormalizedString() : version.ToString();
this.Logger.LogWarning(
"Couldn't satisfy lookup for {Version}. Falling back to first found component for {MatchingLibraryName}, resolving to version {MatchingLibraryVersion}.",
versionString,
matchingLibrary.Name,
matchingLibrary.Version);
}
return matchingLibrary;
}
}

Просмотреть файл

@ -53,17 +53,32 @@ public sealed class NuGetPackagesConfigDetector : FileComponentDetector
{
try
{
var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
var packagesConfig = new PackagesConfigReader(processRequest.ComponentStream.Stream);
foreach (var package in packagesConfig.GetPackages(allowDuplicatePackageIds: true))
{
processRequest.SingleFileComponentRecorder.RegisterUsage(
new DetectedComponent(
var detectedComponent = new DetectedComponent(
new NuGetComponent(
package.PackageIdentity.Id,
package.PackageIdentity.Version.ToNormalizedString())),
package.PackageIdentity.Version.ToNormalizedString()));
singleFileComponentRecorder.RegisterUsage(
detectedComponent,
true,
null,
/* TODO: Is this really the same concept?
Docs for NuGet say packages.config development dependencies are just not persisted as dependencies in the package.
That is not same as excluding from the output directory / runtime. */
package.IsDevelopmentDependency);
// get the actual component in case it already exists
var libraryComponent = singleFileComponentRecorder.GetComponent(detectedComponent.Component.Id);
// Add framework information to the actual component
if (package.TargetFramework is not null)
{
((NuGetComponent)libraryComponent.Component).TargetFrameworks.Add(package.TargetFramework.GetShortFolderName());
}
}
}
catch (Exception e) when (e is PackagesConfigReaderException or XmlException)

Просмотреть файл

@ -278,6 +278,15 @@ public class NuGetProjectModelProjectCentricComponentDetector : FileComponentDet
var libraryComponent = new DetectedComponent(new NuGetComponent(library.Name, library.Version.ToNormalizedString()));
singleFileComponentRecorder.RegisterUsage(libraryComponent, explicitlyReferencedComponentIds.Contains(libraryComponent.Component.Id), parentComponentId);
// get the actual component in case it already exists
libraryComponent = singleFileComponentRecorder.GetComponent(libraryComponent.Component.Id);
// Add framework information to the actual component
if (target.TargetFramework is not null)
{
((NuGetComponent)libraryComponent.Component).TargetFrameworks.Add(target.TargetFramework.GetShortFolderName());
}
foreach (var dependency in library.Dependencies)
{
if (visited.Contains(dependency.Id))

Просмотреть файл

@ -0,0 +1,22 @@
namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Detectors.NuGet;
/// <summary>
/// Validating the <see cref="NuGetTargetFrameworkExperiment"/>.
/// </summary>
public class NuGetTargetFrameworkExperiment : IExperimentConfiguration
{
/// <inheritdoc />
public string Name => "NuGetTargetFrameworkAwareDetector";
/// <inheritdoc />
public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is NuGetProjectModelProjectCentricComponentDetector;
/// <inheritdoc />
public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is NuGetPackageReferenceFrameworkAwareDetector;
/// <inheritdoc />
public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true;
}

Просмотреть файл

@ -63,6 +63,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IExperimentProcessor, DefaultExperimentProcessor>();
services.AddSingleton<IExperimentConfiguration, SimplePipExperiment>();
services.AddSingleton<IExperimentConfiguration, RustCliDetectorExperiment>();
services.AddSingleton<IExperimentConfiguration, NuGetTargetFrameworkExperiment>();
// Detectors
// CocoaPods
@ -104,6 +105,7 @@ public static class ServiceCollectionExtensions
services.AddSingleton<IComponentDetector, NuGetComponentDetector>();
services.AddSingleton<IComponentDetector, NuGetPackagesConfigDetector>();
services.AddSingleton<IComponentDetector, NuGetProjectModelProjectCentricComponentDetector>();
services.AddSingleton<IComponentDetector, NuGetPackageReferenceFrameworkAwareDetector>();
// PIP
services.AddSingleton<IPyPiClient, PyPiClient>();

Просмотреть файл

@ -0,0 +1,312 @@
namespace Microsoft.ComponentDetection.Detectors.Tests;
using System;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.ComponentDetection.Contracts;
using Microsoft.ComponentDetection.Contracts.TypedComponent;
using Microsoft.ComponentDetection.Detectors.NuGet;
using Microsoft.ComponentDetection.Detectors.Tests.Mocks;
using Microsoft.ComponentDetection.Detectors.Tests.Utilities;
using Microsoft.ComponentDetection.TestsUtilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
[TestClass]
[TestCategory("Governance/All")]
[TestCategory("Governance/ComponentDetection")]
public class NuGetPackageReferenceFrameworkAwareDetectorTests : BaseDetectorTest<NuGetPackageReferenceFrameworkAwareDetector>
{
private readonly string projectAssetsJsonFileName = "project.assets.json";
private readonly Mock<IFileUtilityService> fileUtilityServiceMock;
public NuGetPackageReferenceFrameworkAwareDetectorTests()
{
this.fileUtilityServiceMock = new Mock<IFileUtilityService>();
this.fileUtilityServiceMock.Setup(x => x.Exists(It.IsAny<string>()))
.Returns(true);
this.DetectorTestUtility.AddServiceMock(this.fileUtilityServiceMock);
}
[TestMethod]
public async Task ScanDirectoryAsync_Base_2_2_VerificationAsync()
{
var osAgnostic = this.Convert22SampleToOSAgnostic(TestResources.project_assets_2_2);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var detectedComponents = componentRecorder.GetDetectedComponents();
// Number of unique nodes in ProjectAssetsJson
Console.WriteLine(string.Join(",", detectedComponents.Select(x => x.Component.Id)));
detectedComponents.Should().HaveCount(22);
var nonDevComponents = detectedComponents.Where(c => !componentRecorder.GetEffectiveDevDependencyValue(c.Component.Id).GetValueOrDefault());
nonDevComponents.Should().HaveCount(3);
detectedComponents.Select(x => x.Component).Cast<NuGetComponent>().FirstOrDefault(x => x.Name.Contains("coverlet.msbuild")).Should().NotBeNull();
componentRecorder.ForAllComponents(grouping => grouping.AllFileLocations.Should().Contain(location => location.Contains("Loader.csproj")));
}
[TestMethod]
public async Task ScanDirectoryAsync_Base_2_2_additional_VerificationAsync()
{
var osAgnostic = this.Convert22SampleToOSAgnostic(TestResources.project_assets_2_2_additional);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var detectedComponents = componentRecorder.GetDetectedComponents();
// Number of unique nodes in ProjectAssetsJson
Console.WriteLine(string.Join(",", detectedComponents.Select(x => x.Component.Id)));
detectedComponents.Should().HaveCount(68);
var nonDevComponents = detectedComponents.Where(c => !componentRecorder.GetEffectiveDevDependencyValue(c.Component.Id).GetValueOrDefault());
nonDevComponents.Should().HaveCount(23);
nonDevComponents.Select(x => x.Component).Cast<NuGetComponent>().FirstOrDefault(x => x.Name.Contains("Polly")).Should().NotBeNull();
nonDevComponents.Select(x => x.Component).Cast<NuGetComponent>().Count(x => x.Name.Contains("System.Composition")).Should().Be(5);
var nugetVersioning = nonDevComponents.FirstOrDefault(x => (x.Component as NuGetComponent).Name.Contains("NuGet.DependencyResolver.Core"));
nugetVersioning.Should().NotBeNull();
componentRecorder.IsDependencyOfExplicitlyReferencedComponents<NuGetComponent>(
nugetVersioning.Component.Id,
x => x.Name.Contains("NuGet.ProjectModel")).Should().BeTrue();
componentRecorder.ForAllComponents(grouping => grouping.AllFileLocations.Should().Contain(location => location.Contains("Detectors.csproj")));
}
[TestMethod]
public async Task ScanDirectoryAsync_ExcludedFrameworkComponent_2_2_VerificationAsync()
{
var osAgnostic = this.Convert22SampleToOSAgnostic(TestResources.project_assets_2_2);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var developmentDependencies = componentRecorder.GetDetectedComponents().Where(c => componentRecorder.GetEffectiveDevDependencyValue(c.Component.Id).GetValueOrDefault());
developmentDependencies.Should().HaveCountGreaterThan(5, "Ommitted framework assemblies are missing. There should be more than ten, but this is a gut check to make sure we have data.");
developmentDependencies.Should().Contain(c => c.Component.Id.StartsWith("Microsoft.NETCore.App "), "Microsoft.NETCore.App should be treated as a development dependency.");
}
[TestMethod]
public async Task ScanDirectoryAsync_DependencyGraph_2_2_additional_VerificationAsync()
{
var osAgnostic = this.Convert22SampleToOSAgnostic(TestResources.project_assets_2_2_additional);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var graphsByLocation = componentRecorder.GetDependencyGraphsByLocation();
var graph = graphsByLocation.Values.First();
var expectedDependencyIdsForCompositionTypedParts = new[]
{
"NuGet.DependencyResolver.Core 5.6.0 - NuGet",
};
var detectedComponents = componentRecorder.GetDetectedComponents();
var componentDetectionCommon = detectedComponents.First(x => x.Component.Id.Contains("NuGet.ProjectModel"));
var dependencies = graph.GetDependenciesForComponent(componentDetectionCommon.Component.Id);
foreach (var expectedId in expectedDependencyIdsForCompositionTypedParts)
{
dependencies.Should().Contain(expectedId);
}
expectedDependencyIdsForCompositionTypedParts.Should().HaveSameCount(dependencies);
detectedComponents.Should().HaveSameCount(graph.GetComponents());
// Top level dependencies look like this:
// (we expect all non-proj and non-framework to show up as explicit refs, so those will be absent from the check)
//
// "DotNet.Glob >= 2.1.1",
// "Microsoft.NETCore.App >= 2.2.8",
// "Microsoft.VisualStudio.Services.Governance.ComponentDetection.Common >= 1.0.0",
// "Microsoft.VisualStudio.Services.Governance.ComponentDetection.Contracts >= 1.0.0",
// "MinVer >= 2.5.0",
// "Nett >= 0.10.0",
// "Newtonsoft.Json >= 12.0.3",
// "NuGet.ProjectModel >= 5.6.0",
// "NuGet.Versioning >= 5.6.0",
// "Polly >= 7.0.3",
// "SemanticVersioning >= 1.2.0",
// "StyleCop.Analyzers >= 1.0.2",
// "System.Composition.AttributedModel >= 1.4.0",
// "System.Composition.Convention >= 1.4.0",
// "System.Composition.Hosting >= 1.4.0",
// "System.Composition.Runtime >= 1.4.0",
// "System.Composition.TypedParts >= 1.4.0",
// "System.Reactive >= 4.1.2",
// "System.Threading.Tasks.Dataflow >= 4.9.0",
// "coverlet.msbuild >= 2.5.1",
// "yamldotnet >= 5.3.0"
var expectedExplicitRefs = new[]
{
"DotNet.Glob",
"Microsoft.NETCore.App",
"MinVer",
"Nett",
"Newtonsoft.Json",
"NuGet.ProjectModel",
"NuGet.Versioning",
"Polly",
"SemanticVersioning",
"StyleCop.Analyzers",
"System.Composition.AttributedModel",
"System.Composition.Convention",
"System.Composition.Hosting",
"System.Composition.Runtime",
"System.Composition.TypedParts",
"System.Reactive",
"System.Threading.Tasks.Dataflow",
"coverlet.msbuild",
"YamlDotNet",
};
foreach (var componentId in graph.GetComponents())
{
var component = detectedComponents.First(x => x.Component.Id == componentId);
var expectedExplicitRefValue = expectedExplicitRefs.Contains(((NuGetComponent)component.Component).Name);
graph.IsComponentExplicitlyReferenced(componentId).Should().Be(expectedExplicitRefValue, "{0} should{1} be explicitly referenced.", componentId, expectedExplicitRefValue ? string.Empty : "n't");
}
}
[TestMethod]
public async Task ScanDirectoryAsync_Base_3_1_VerificationAsync()
{
var osAgnostic = this.Convert31SampleToOSAgnostic(TestResources.project_assets_3_1);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
// Number of unique nodes in ProjectAssetsJson
var detectedComponents = componentRecorder.GetDetectedComponents();
detectedComponents.Should().HaveCount(11);
var nonDevComponents = detectedComponents.Where(c => !componentRecorder.GetEffectiveDevDependencyValue(c.Component.Id).GetValueOrDefault());
nonDevComponents.Should().ContainSingle();
nonDevComponents.Select(x => x.Component).Cast<NuGetComponent>().Single().Name.Should().StartWith("Microsoft.Extensions.DependencyModel");
var systemTextJson = detectedComponents.FirstOrDefault(x => (x.Component as NuGetComponent).Name.Contains("System.Text.Json"));
componentRecorder.IsDependencyOfExplicitlyReferencedComponents<NuGetComponent>(
systemTextJson.Component.Id,
x => x.Name.Contains("Microsoft.Extensions.DependencyModel")).Should().BeTrue();
componentRecorder.ForAllComponents(grouping => grouping.AllFileLocations.Should().Contain(location => location.Contains("ExtCore.WebApplication.csproj")));
}
[TestMethod]
public async Task ScanDirectoryAsync_ExcludedFrameworkComponent_3_1_VerificationAsync()
{
var osAgnostic = this.Convert31SampleToOSAgnostic(TestResources.project_assets_3_1);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var developmentDependencies = componentRecorder.GetDetectedComponents().Where(c => componentRecorder.GetEffectiveDevDependencyValue(c.Component.Id).GetValueOrDefault());
developmentDependencies.Should().HaveCount(10, "Omitted framework assemblies are missing.");
developmentDependencies.Should().Contain(c => c.Component.Id.StartsWith("System.Reflection "), "System.Reflection should be treated as a development dependency.");
}
[TestMethod]
public async Task ScanDirectoryAsync_DependencyGraph_3_1_VerificationAsync()
{
var osAgnostic = this.Convert31SampleToOSAgnostic(TestResources.project_assets_3_1);
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
var graphsByLocation = componentRecorder.GetDependencyGraphsByLocation();
var graph = graphsByLocation.Values.First();
var expectedDependencyIdsForExtensionsDependencyModel = new[]
{
"System.Text.Json 4.6.0 - NuGet",
};
var detectedComponents = componentRecorder.GetDetectedComponents();
var componentDetectionCommon = detectedComponents.First(x => x.Component.Id.Contains("Microsoft.Extensions.DependencyModel"));
var dependencies = graph.GetDependenciesForComponent(componentDetectionCommon.Component.Id);
foreach (var expectedId in expectedDependencyIdsForExtensionsDependencyModel)
{
dependencies.Should().Contain(expectedId);
}
detectedComponents.Should().HaveSameCount(graph.GetComponents());
// Top level dependencies look like this:
// (we expect all non-proj and non-framework to show up as explicit refs, so those will be absent from the check)
//
// "Microsoft.Extensions.DependencyModel >= 3.0.0",
// "System.Runtime.Loader >= 4.3.0"
var expectedExplicitRefs = new[]
{
"Microsoft.Extensions.DependencyModel",
"System.Runtime.Loader",
};
foreach (var componentId in graph.GetComponents())
{
var component = detectedComponents.First(x => x.Component.Id == componentId);
var expectedExplicitRefValue = expectedExplicitRefs.Contains(((NuGetComponent)component.Component).Name);
graph.IsComponentExplicitlyReferenced(componentId).Should().Be(expectedExplicitRefValue, "{0} should{1} be explicitly referenced.", componentId, expectedExplicitRefValue ? string.Empty : "n't");
}
}
[TestMethod]
public async Task ScanDirectory_NoPackageSpecAsync()
{
var osAgnostic =
@"{
""version"": 3,
""targets"": {
"".NETCoreApp,Version=v3.0"": {}
},
""packageFolders"": {}
}";
var (scanResult, componentRecorder) = await this.DetectorTestUtility
.WithFile(this.projectAssetsJsonFileName, osAgnostic)
.ExecuteDetectorAsync();
scanResult.ResultCode.Should().Be(ProcessingResultCode.Success);
var dependencyGraphs = componentRecorder.GetDependencyGraphsByLocation();
dependencyGraphs.Should().BeEmpty();
}
private string Convert22SampleToOSAgnostic(string project_assets)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return project_assets;
}
project_assets = project_assets.Replace("D:\\\\Source\\\\componentdetection-bcde\\\\src\\\\Common\\\\", "/d/source/componentdetection-bcde/src/Common/");
project_assets = project_assets.Replace("D:\\\\Source\\\\componentdetection-bcde\\\\src\\\\Contracts\\\\", "/d/source/componentdetection-bcde/src/Contracts/");
project_assets = project_assets.Replace("D:\\\\Source\\\\componentdetection-bcde\\\\src\\\\Detectors\\\\", "/d/source/componentdetection-bcde/src/Detectors/");
project_assets = project_assets.Replace("D:\\\\Source\\\\componentdetection-bcde\\\\src\\\\Orchestrator\\\\", "/d/source/componentdetection-bcde/src/Orchestrator/");
project_assets = project_assets.Replace("D:\\\\Source\\\\componentdetection-bcde\\\\src\\\\Loader\\\\", "/d/source/componentdetection-bcde/src/Loader/");
return project_assets;
}
private string Convert31SampleToOSAgnostic(string project_assets)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return project_assets;
}
project_assets = project_assets.Replace("D:\\\\Source\\\\ExtCore\\\\src\\\\ExtCore.WebApplication\\\\", "/d/Source/ExtCore/src/ExtCore.WebApplication/");
project_assets = project_assets.Replace("D:\\\\Source\\\\ExtCore\\\\src\\\\ExtCore.Infrastructure\\\\", "/d/Source/ExtCore/src/ExtCore.Infrastructure/");
return project_assets;
}
}