Merged PR 643482: Fix runtime failures in DropDaemon's GetSbomPackages

- Remove the `__ENABLE_SBOM_PACKAGE_CONVERSION` env variable. This was meant to guard against using this feature while the library had a bug that is already resolved, so not necessary anymore
- Microsoft.SBOM.Adapters and some other packages reference Newtonsoft.Json 13.0.1, while we were using 12.0.3 throughout. This was causing a failure when the runtime tried to load the assembly after calling the library. Version 13.0.1 is incompatible with earlier ones so a downgrading binding redirect is not possible. Instead, we deploy the drop daemon with Newtonsoft.Json version 13.0.1 and use "forward" binding redirects.
- Add some extra logging

With these changes, an end-to-end build where CG is run in the build runner and the packages are retrieved and added to the SPDX SBOM was successful: https://cbtest.microsoft.com/build/1d85303e-322e-4ddf-af1f-585ecce96079
This commit is contained in:
Marcelo Lynch 🧉 2022-01-06 01:28:25 +00:00
Родитель 46030cb8d4
Коммит 1d0dc08a64
7 изменённых файлов: 73 добавлений и 63 удалений

Просмотреть файл

@ -71,10 +71,6 @@ namespace Tool.DropDaemon
private readonly ISBOMGenerator m_sbomGenerator;
private BsiMetadataExtractor m_bsiMetadataExtractor;
private readonly string m_sbomGenerationOutputDirectory;
/// <summary>
/// If set to 1, SBOMPackages will be added from component detection data to the SBOM.
/// </summary>
private const string m_enableSBOMPackageConversion = "__ENABLE_SBOM_PACKAGE_CONVERSION";
// This field should be removed once this SBOM format is deprecated
// Related work item: #1895958.
@ -884,63 +880,60 @@ namespace Tool.DropDaemon
/// </returns>
private IEnumerable<SBOMPackage> GetSbomPackages()
{
var shouldConvertPackages = Environment.GetEnvironmentVariable(m_enableSBOMPackageConversion);
if (shouldConvertPackages != null && shouldConvertPackages == "1")
// Read Path for bcde output from environment, this should already be set by Cloudbuild
var bcdeOutputJsonPath = Environment.GetEnvironmentVariable(Constants.ComponentGovernanceBCDEOutputFilePath);
if (string.IsNullOrWhiteSpace(bcdeOutputJsonPath))
{
// Read Path for bcde output from environment, this should already be set by Cloudbuild
var bcdeOutputJsonPath = Environment.GetEnvironmentVariable(Constants.ComponentGovernanceBCDEOutputFilePath);
if (string.IsNullOrWhiteSpace(bcdeOutputJsonPath))
{
// This shouldn't happen, but SBOM creation can still happen without it a set of packages. So, log it and return an empty set.
// TODO [pgunasekara]: Change this to a Warning. Currently this is only Info level until CB changes are fully rolled out to avoid generating warnings unnecessarily.
Logger.Info($"The '{Constants.ComponentGovernanceBCDEOutputFilePath}' environment variable was not found. Component detection data will not be included in build manifest.");
return new List<SBOMPackage>();
}
else if (!System.IO.File.Exists(bcdeOutputJsonPath))
{
Logger.Warning($"Component detection output file not found at path '{bcdeOutputJsonPath}'. Component detection data will not be included in build manifest.");
return new List<SBOMPackage>();
}
var (adapterReport, packages) = new ComponentDetectionToSBOMPackageAdapter().TryConvert(bcdeOutputJsonPath);
foreach (var reportItem in adapterReport.Report)
{
switch (reportItem.Type)
{
case AdapterReportItemType.Success:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Info("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
case AdapterReportItemType.Warning:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Warning("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
case AdapterReportItemType.Failure:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Error("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
}
}
return packages ?? new List<SBOMPackage>();
// This shouldn't happen, but SBOM creation can still happen without it a set of packages. So, log it and return an empty set.
// TODO [pgunasekara]: Change this to a Warning. Currently this is only Info level until CB changes are fully rolled out to avoid generating warnings unnecessarily.
Logger.Info($"[GetSbomPackages] The '{Constants.ComponentGovernanceBCDEOutputFilePath}' environment variable was not found. Component detection data will not be included in build manifest.");
return new List<SBOMPackage>();
}
else if (!System.IO.File.Exists(bcdeOutputJsonPath))
{
Logger.Warning($"[GetSbomPackages] Component detection output file not found at path '{bcdeOutputJsonPath}'. Component detection data will not be included in build manifest.");
return new List<SBOMPackage>();
}
return new List<SBOMPackage>();
Logger.Info($"[GetSbomPackages] Retrieving component detection package list from file at {bcdeOutputJsonPath}");
var (adapterReport, packages) = new ComponentDetectionToSBOMPackageAdapter().TryConvert(bcdeOutputJsonPath);
foreach (var reportItem in adapterReport.Report)
{
switch (reportItem.Type)
{
case AdapterReportItemType.Success:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Info("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
case AdapterReportItemType.Warning:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Warning("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
case AdapterReportItemType.Failure:
{
if (!string.IsNullOrEmpty(reportItem.Details))
{
Logger.Error("[ComponentDetectionToSBOMPackageAdapter] " + reportItem.Details);
}
break;
}
}
}
var result = packages ?? new List<SBOMPackage>();
Logger.Verbose($"[GetSbomPackages] Retrieved {result.Count()} packages");
return result;
}
/// <summary>

Просмотреть файл

@ -45,7 +45,6 @@ export namespace DropDaemon {
importFrom("Microsoft.VisualStudio.Services.BlobStore.Client").pkg,
importFrom("Microsoft.VisualStudio.Services.Client").pkg,
importFrom("Microsoft.VisualStudio.Services.InteractiveClient").pkg,
importFrom("Newtonsoft.Json").pkg,
importFrom("WindowsAzure.Storage").pkg,
importFrom("Microsoft.Azure.Storage.Common").pkg,
importFrom("Microsoft.Extensions.Logging.Abstractions.v6.0.0").pkg,
@ -64,6 +63,7 @@ export namespace DropDaemon {
),
importFrom("Microsoft.SBOM.Adapters").withQualifier({ targetFramework: "netstandard2.0" }).pkg,
importFrom("System.Text.Json.v5.0.0").pkg,
importFrom("Newtonsoft.Json.v13.0.1").pkg,
importFrom("System.Text.Encodings.Web.v5.0.1").pkg,
],
internalsVisibleTo: [
@ -120,6 +120,13 @@ export namespace DropDaemon {
export function dropDaemonBindingRedirects() {
return [
...BuildXLSdk.cacheBindingRedirects(),
{
name: "Newtonsoft.Json",
publicKeyToken: "30ad4fe6b2a6aeed",
culture: "neutral",
oldVersion: "0.0.0.0-13.0.0.0",
newVersion: "13.0.0.0", // Corresponds to { id: "Newtonsoft.Json", version: "13.0.1", alias: "Newtonsoft.Json.v13.0.1" }
},
{
name: "System.Text.Json",
publicKeyToken: "cc7b13ffcd2ddd51",
@ -141,6 +148,7 @@ export namespace DropDaemon {
export function dropDaemonRuntimeContentToSkip() {
return [
importFrom("System.Text.Json").withQualifier({ targetFramework: "netstandard2.0" }).pkg,
importFrom("Newtonsoft.Json").pkg,
importFrom("System.Text.Encodings.Web").withQualifier({ targetFramework: "netstandard2.0" }).pkg,
importFrom("Microsoft.Extensions.Logging.Abstractions").pkg,
];

Просмотреть файл

@ -549,7 +549,6 @@ export namespace DropDaemonRunner {
"__CLOUDBUILD_AUTH_HELPER_CONFIG__",
"QAUTHMATERIALROOT", // Auth material for low-privilege build.
"AZURE_ARTIFACTS_CREDENTIALPROVIDERS_PATH", // Cloudbuild auth helper executable path for build cache, symbol, and drop
"__ENABLE_SBOM_PACKAGE_CONVERSION",
...cloudBuildVarsPointingToDirs];
/**
* Sets the values of the 'forwardEnvironmentVars'

Просмотреть файл

@ -47,7 +47,7 @@
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-12.0.0.0" newVersion="12.0.0.0" />
<bindingRedirect oldVersion="0.0.0.0-13.0.0.0" newVersion="13.0.0.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>

Просмотреть файл

@ -2755,6 +2755,15 @@
}
}
},
{
"Component": {
"Type": "NuGet",
"NuGet": {
"Name": "Newtonsoft.Json",
"Version": "13.0.1"
}
}
},
{
"Component": {
"Type": "NuGet",

Просмотреть файл

@ -76,8 +76,8 @@ export const pkgs = isMicrosoftInternal ? [
{ id: "Microsoft.SBOMCore", version: sbomApiVersion, dependentPackageIdsToSkip: ["Microsoft.Extensions.Logging.Abstractions"] },
{ id: "Microsoft.Parsers.ManifestGenerator", version: sbomApiVersion, dependentPackageIdsToSkip: ["Newtonsoft.Json"]},
{ id: "Microsoft.Parsers.SPDX22SBOMParser", version: sbomApiVersion },
{ id: "Microsoft.SBOM.Adapters", version: sbomApiVersion },
{ id: "Microsoft.ComponentDetection.Contracts", version: "1.0.8" },
{ id: "Microsoft.SBOM.Adapters", version: sbomApiVersion, dependentPackageIdsToSkip : ["Newtonsoft.Json"] },
{ id: "Microsoft.ComponentDetection.Contracts", version: "1.0.8", dependentPackageIdsToSkip: ["Newtonsoft.Json"] },
{ id: "Microsoft.ManifestInterface", version: sbomApiVersion, dependentPackageIdsToSkip: ["System.Text.Json"] },
{ id: "Microsoft.Sbom.Contracts", version: sbomApiVersion },
{ id: "Microsoft.Bcl.HashCode", version: "1.1.1" },

Просмотреть файл

@ -259,6 +259,7 @@ export const pkgs = [
dependentPackageIdsToSkip: ["System.Memory", "System.Buffers", "System.ValueTuple", "System.Runtime.CompilerServices.Unsafe", "System.Numerics.Vectors", "System.Threading.Tasks.Extensions", "Microsoft.Bcl.AsyncInterfaces", "System.Text.Encodings.Web"],
alias: "System.Text.Json.v5.0.0"
},
{ id: "Newtonsoft.Json", version: "13.0.1", alias: "Newtonsoft.Json.v13.0.1" },
{ id: "System.Threading.AccessControl", version: pkgVersionNext },
{ id: "System.IO.FileSystem.AccessControl", version: pkgVersion6Preview, alias: "System.IO.FileSystem.AccessControl.v6.0.0" },