Component Governance for NuGet Packages (#889)

* Skeleton code for generating cgmanifest.json for nugets

* some unit tests

* remove unused using

* Sorted cgmanifest json generation complete

* cgmanifest.json write partial changes

* cgmanifest.json generation and validation complete

* Added Validate cgmanifest steps

* cgmanifest validation eroor messages updated

* Added help text

* Unit Test for cgmanifet complete

* Code clean up

* Merge fix

* Added missing git package file for component governance

* Suggested Comment fixes

* Added error logs before all CG Manifest build failures

* nit changes

* Unit Test path changes for Mac

Public/Src/App/Bxl/Args.cs

@@ -473,6 +473,12 @@ namespace BuildXL
                         OptionHandlerFactory.CreateBoolOption(
                             "forceUseEngineInfoFromCache",
                             sign => schedulingConfiguration.ForceUseEngineInfoFromCache = sign),
+                        OptionHandlerFactory.CreateOption(
+                            "generateCgManifestForNugets",
+                            opt => frontEndConfiguration.GenerateCgManifestForNugets = CommandLineUtilities.ParsePathOption(opt, pathTable)),
+                        OptionHandlerFactory.CreateOption(
+                            "validateCgManifestForNugets",
+                            opt => frontEndConfiguration.ValidateCgManifestForNugets = CommandLineUtilities.ParsePathOption(opt, pathTable)),
                         OptionHandlerFactory.CreateBoolOption(
                             "hardExitOnErrorInDetours",
                             sign => sandboxConfiguration.HardExitOnErrorInDetours = sign),

Public/Src/App/Bxl/HelpText.cs

@@ -1019,6 +1019,17 @@ namespace BuildXL
 
             #endregion
 
+            #region Component Governance Manifest 
+            hw.WriteBanner(Strings.HelpText_DisplayHelp_CgManifestBanner);
+
+            hw.WriteOption("/generateCgManifestForNugest:<file>",
+                Strings.HelpText_DisplayHelp_GenerateCgManifest);
+
+            hw.WriteOption("/validateCgManifestForNugest:<file>",
+                Strings.HelpText_DisplayHelp_ValidateCgManifest);
+
+            #endregion
+
             hw.WriteBanner(Strings.HelpText_DisplayHelp_MsBuildBanner);
 
             #region MSBuild

Public/Src/App/Bxl/Strings.resx

@@ -1027,4 +1027,13 @@ Example: ad2d42d2ec5d2ca0c0b7ad65402d07c7ef40b91e</value>
   <data name="HelpText_DisplayHelp_AugmentingPathSetCommonalityFactor" xml:space="preserve">
     <value>Used to compute the number of times (i.e. {augmentingPathSetCommonalityFactor} * {pathSetThreshold}) an entry must appear among paths in the observed path set in order to be included in the common path set. Value must be (0, 1]</value>
   </data>
+  <data name="HelpText_DisplayHelp_CgManifestBanner" xml:space="preserve">
+    <value>Component Governance</value>
+  </data>
+  <data name="HelpText_DisplayHelp_GenerateCgManifest" xml:space="preserve">
+    <value>Generates a cgmanifest.json file at the specified path. This file contains the names and versions for all Nuget packages used within BuildXL, and is used for Component Governance during CloudBuild.</value>
+  </data>
+  <data name="HelpText_DisplayHelp_ValidateCgManifest" xml:space="preserve">
+    <value>Validates the cgmanifest.json file at the specified path. This file should contain up-to-date names and versions of all Nuget packages used within BuildXL for Component Governance. Any mismatch will cause the Build to fail. Updated file can be created using the /generateCgManifestForNugets:&lt;path&gt;</value>
+  </data>
 </root>

Public/Src/FrontEnd/Nuget/BuildXL.FrontEnd.Nuget.dsc

@@ -39,6 +39,7 @@ namespace Nuget {
             importFrom("BuildXL.Utilities").Storage.dll,
             importFrom("BuildXL.Utilities").Script.Constants.dll,
 
+            importFrom("Newtonsoft.Json").pkg,
             importFrom("NuGet.Versioning").pkg,
 
             ...BuildXLSdk.tplPackages,

Public/Src/FrontEnd/Nuget/NugetAnalyzedPackage.cs

@@ -70,6 +70,9 @@ namespace BuildXL.FrontEnd.Nuget
         /// <nodoc />
         public string Id => PackageOnDisk.Package.Id;
 
+        /// <nodoc />
+        public string NugetName { get; set; }
+
         /// <nodoc />
         public string Alias => PackageOnDisk.Package.Alias;
 

Public/Src/FrontEnd/Nuget/NugetCgManifestGenerator.cs

@@ -0,0 +1,102 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Linq;
+using BuildXL.FrontEnd.Sdk;
+using BuildXL.Utilities.Collections;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace BuildXL.FrontEnd.Nuget
+{
+    /// <summary>
+    /// NugetCgManifestGenerator is used for creation and comparasion of a manifest file for Component Governance.
+    /// The cgmanifest file contains information about all the Nuget Packages used in BuildXL with all their versions in use
+    /// The cgmanifest file is used by Component Governance to determine security risks within components used by BuildXL
+    /// This manifest file will only be picked up for Component Governance if it is named "cgmanifest.json" as per cg documentation: https://docs.opensource.microsoft.com/tools/cg.html
+    /// </summary>
+    public sealed class NugetCgManifestGenerator
+    {
+        private FrontEndContext Context { get; }
+
+        /// <nodoc />
+        public NugetCgManifestGenerator(FrontEndContext context)
+        {
+            Context = context;
+        }
+
+        /// <summary>
+        /// Generates json as an indented string containing all the NuGet package names and versions used in BuildXL
+        /// </summary>
+        public string GenerateCgManifestForPackages(MultiValueDictionary<string, Package> packages)
+        {
+            var components = packages
+                .Keys
+                .SelectMany(nugetName => packages[nugetName].Select(package => new NugetPackageAndVersionStore(nugetName, ExtractNugetVersion(package))))
+                .OrderBy(c => c.Name)
+                .ThenBy(c => c.Version)
+                .Select(c => ToNugetComponent(c.Name, c.Version))
+                .ToList();
+
+            var cgmanifest = new
+            {
+                Version = 1,
+                Registrations = components
+            };
+
+            return JsonConvert.SerializeObject(cgmanifest, Formatting.Indented);
+        }
+
+        /// <summary>
+        /// Compares <paramref name="lhsManifest"/> and <paramref name="rhsManifest"/> for equality;
+        /// returns true if they are equal, and false otherwise. 
+        /// 
+        /// This equality check is case-insensitive and white space agnostic.
+        /// </summary>
+        public bool CompareForEquality(string lhsManifest, string rhsManifest)
+        {
+            try
+            {
+                return JToken.DeepEquals(JObject.Parse(lhsManifest), JObject.Parse(rhsManifest));
+            }
+            catch (JsonReaderException)
+            {
+                // The existing Manifest file was in invalid JSON format.
+                // Hence it does not match.
+                return false;
+            }
+            
+        }
+
+        private string ExtractNugetVersion(Package p)
+        {
+            return p.Path.GetParent(Context.PathTable).GetName(Context.PathTable).ToString(Context.StringTable);
+        }
+
+        private object ToNugetComponent(string name, string version)
+        {
+            return new
+            {
+                Component = new
+                {
+                    Type = "NuGet",
+                    NuGet = new
+                    {
+                        Name = name,
+                        Version = version
+                    }
+                }
+            };
+        }
+
+        private class NugetPackageAndVersionStore {
+            public string Name { get; }
+            public string Version { get; }
+
+            public NugetPackageAndVersionStore(string name, string version) {
+                Name = name;
+                Version = version;
+            }
+        }
+    }
+}

Public/Src/FrontEnd/Nuget/NugetResolver.cs

@@ -2,13 +2,18 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Diagnostics.ContractsLight;
+using System.IO;
+using System.Linq;
+using System.Text;
 using System.Threading.Tasks;
 using BuildXL.FrontEnd.Nuget;
 using BuildXL.FrontEnd.Script.Evaluator;
 using BuildXL.FrontEnd.Script.Tracing;
 using BuildXL.FrontEnd.Script.Values;
 using BuildXL.FrontEnd.Sdk;
+using BuildXL.Native.IO;
 using BuildXL.Utilities;
 using BuildXL.Utilities.Configuration;
 using static BuildXL.Utilities.FormattableStringEx;
@@ -20,6 +25,7 @@ namespace BuildXL.FrontEnd.Script
     /// </summary>
     public sealed class NugetResolver : DScriptSourceResolver
     {
+        internal const string CGManifestResolverName = "CGManifestGenerator";
         private WorkspaceNugetModuleResolver m_nugetWorkspaceResolver;
 
         /// <nodoc />
@@ -35,6 +41,7 @@ namespace BuildXL.FrontEnd.Script
         { }
 
         /// <inheritdoc/>
+        [SuppressMessage("AsyncUsage", "AsyncFixer02:awaitinsteadofwait")]
         public override async Task<bool> InitResolverAsync(IResolverSettings resolverSettings, object workspaceResolver)
         {
             Contract.Requires(resolverSettings != null);
@@ -62,12 +69,89 @@ namespace BuildXL.FrontEnd.Script
 
             m_owningModules = new Dictionary<ModuleId, Package>();
 
-            foreach (var package in maybePackages.Result)
+            foreach (var package in maybePackages.Result.Values.SelectMany(v => v))
             {
                 m_packages[package.Id] = package;
                 m_owningModules[package.ModuleId] = package;
             }
 
+            if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid ||
+                Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid)
+            {
+                var cgManfiestGenerator = new NugetCgManifestGenerator(Context);
+                string generatedCgManifest = cgManfiestGenerator.GenerateCgManifestForPackages(maybePackages.Result);
+                string existingCgManifest = "INVALID";
+
+                if ( !Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid &&
+                      Configuration.FrontEnd.ValidateCgManifestForNugets.IsValid )
+                {
+                    // Validation of existing cgmainfest.json results in failure due to mismatch. Should fail the build in this case.
+                    try
+                    {
+                        existingCgManifest = File.ReadAllText(Configuration.FrontEnd.ValidateCgManifestForNugets.ToString(Context.PathTable));
+                        FrontEndHost.Engine.RecordFrontEndFile(
+                            Configuration.FrontEnd.ValidateCgManifestForNugets,
+                            CGManifestResolverName);
+                    }
+                    // CgManifest FileNotFound, log error and fail build
+                    catch (DirectoryNotFoundException e)
+                    {
+                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
+                        return false;
+                    }
+                    catch (FileNotFoundException e)
+                    {
+                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, "Cannot read Component Governance Manifest file from disk\n" + e.ToString());
+                        return false;
+                    }
+                    if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
+                    {
+                        Logger.ReportComponentGovernanceValidationError(Context.LoggingContext, @"Existing Component Governance Manifest file is outdated, please generate a new one using the argument /generateCgManifestForNugets:<path>");
+                        return false;
+                    }
+
+                    m_resolverState = State.ResolverInitialized;
+                    return true;
+                }
+
+                // GenerateCgManifestForNugets writes a new file when the old file does not match, hence it will always be valid and does not need validation
+
+                try
+                {
+                    // We are calling FrontEndHost.Engine.RecordFrontEndFile towards the end of this function because we may update this file after the read below
+                    // Updating the file will cause a hash mismatch and the build to fail if this file is read again downstream
+                    existingCgManifest = File.ReadAllText(Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable));
+                }
+                // CgManifest FileNotFound, continue to write the new file
+                // No operations required as the empty existingCgManifest will not match with the newly generated cgManifest
+                catch (DirectoryNotFoundException) { }
+                catch (FileNotFoundException) { }
+
+                if (!cgManfiestGenerator.CompareForEquality(generatedCgManifest, existingCgManifest))
+                {
+                    if (Configuration.FrontEnd.GenerateCgManifestForNugets.IsValid)
+                    {
+                        // Overwrite or create new cgmanifest.json file with updated nuget package and version info
+                        string targetFilePath = Configuration.FrontEnd.GenerateCgManifestForNugets.ToString(Context.PathTable);
+
+                        try
+                        {
+                            FileUtilities.CreateDirectory(Path.GetDirectoryName(targetFilePath));
+                            await FileUtilities.WriteAllTextAsync(targetFilePath, generatedCgManifest, Encoding.UTF8);
+                        }
+                        catch (BuildXLException e)
+                        {
+                            Logger.ReportComponentGovernanceGenerationError(Context.LoggingContext, "Could not write Component Governance Manifest file to disk\n" + e.ToString());
+                            return false;
+                        }
+                    }
+                }
+
+                FrontEndHost.Engine.RecordFrontEndFile(
+                    Configuration.FrontEnd.GenerateCgManifestForNugets,
+                    CGManifestResolverName);
+            }
+
             m_resolverState = State.ResolverInitialized;
 
             return true;

Public/Src/FrontEnd/Nuget/WorkspaceNugetModuleResolver.cs

@@ -222,18 +222,28 @@ namespace BuildXL.FrontEnd.Nuget
 
         /// <summary>
         /// Returns all packages known by this resolver. This includes potentially embedded packages
+        /// 
+        /// The multi-value dictionary maps the original Nuget package name to (possibly multiple) generated DScript packages.
         /// </summary>
-        public async Task<Possible<IEnumerable<Package>>> GetAllKnownPackagesAsync()
+        public async Task<Possible<MultiValueDictionary<string, Package>>> GetAllKnownPackagesAsync()
         {
             var maybeResult = await DownloadPackagesAndGenerateSpecsIfNeededInternal();
 
             return await maybeResult.ThenAsync(
                 async result =>
-                      {
-                          var maybeDefinitions = await this.GetAllModuleDefinitionsAsync();
-                          return maybeDefinitions.Then(
-                              definitions => definitions.Select(GetPackageForGeneratedProject));
-                      });
+                {
+                    var maybeDefinitions = await this.GetAllModuleDefinitionsAsync();
+                    return maybeDefinitions.Then(
+                        definitions => definitions.Aggregate(
+                            seed: new MultiValueDictionary<string, Package>(),
+                            func: (acc, def) => AddPackage(acc, result.GetOriginalNugetPackageName(def), GetPackageForGeneratedProject(def))));
+                });
+        }
+
+        private MultiValueDictionary<string, Package> AddPackage(MultiValueDictionary<string, Package> dict, string nugetPackageName, Package package)
+        {
+            dict.Add(nugetPackageName, package);
+            return dict;
         }
 
         private Package GetPackageForGeneratedProject(ModuleDefinition moduleDefinition)
@@ -354,7 +364,7 @@ namespace BuildXL.FrontEnd.Nuget
                 m_nugetGenerationResult
                     .GetOrCreate(
                         (@this: this, possiblePackages),
-                        tpl => Task.FromResult(tpl.@this.GetNugetGenerationResultFromDownloadedPackages(tpl.possiblePackages))
+                        tpl => Task.FromResult(tpl.@this.GetNugetGenerationResultFromDownloadedPackages(tpl.possiblePackages, null))
                     )
                     .GetAwaiter()
                     .GetResult()
@@ -498,6 +508,7 @@ namespace BuildXL.FrontEnd.Nuget
                             }
                             else
                             {
+                                packageResult.Result.NugetName = nugetPackage.Package.Id;
                                 restoredPackagesById[packageResult.Result.ActualId] = packageResult.Result;
                             }
                         }
@@ -508,7 +519,7 @@ namespace BuildXL.FrontEnd.Nuget
                         return new WorkspaceModuleResolverGenericInitializationFailure(Kind);
                     }
                 }
-
+                
                 var possiblePackages = GenerateSpecsForDownloadedPackages(restoredPackagesById);
 
                 // At this point we know which are all the packages that contain embedded specs, so we can initialize the embedded resolver properly
@@ -519,7 +530,7 @@ namespace BuildXL.FrontEnd.Nuget
                     return failure;
                 }
 
-                return GetNugetGenerationResultFromDownloadedPackages(possiblePackages);
+                return GetNugetGenerationResultFromDownloadedPackages(possiblePackages, restoredPackagesById);
             }
         }
 
@@ -628,7 +639,8 @@ namespace BuildXL.FrontEnd.Nuget
         }
 
         private Possible<NugetGenerationResult> GetNugetGenerationResultFromDownloadedPackages(
-            Dictionary<string, Possible<AbsolutePath>> possiblePackages)
+            Dictionary<string, Possible<AbsolutePath>> possiblePackages,
+            Dictionary<string, NugetAnalyzedPackage> nugetPackagesByModuleName)
         {
             var generatedProjectsByPackageDescriptor = new Dictionary<ModuleDescriptor, AbsolutePath>(m_resolverSettings.Packages.Count);
             var generatedProjectsByPath = new Dictionary<AbsolutePath, ModuleDescriptor>(m_resolverSettings.Packages.Count);
@@ -653,7 +665,7 @@ namespace BuildXL.FrontEnd.Nuget
                 generatedProjectsByPackageName.Add(moduleDescriptor.Name, moduleDescriptor);
             }
 
-            return new NugetGenerationResult(generatedProjectsByPackageDescriptor, generatedProjectsByPath, generatedProjectsByPackageName);
+            return new NugetGenerationResult(generatedProjectsByPackageDescriptor, generatedProjectsByPath, generatedProjectsByPackageName, nugetPackagesByModuleName);
         }
 
         /// <summary>
@@ -1730,11 +1742,16 @@ namespace BuildXL.FrontEnd.Nuget
     /// </summary>
     internal struct NugetGenerationResult
     {
-        public NugetGenerationResult(Dictionary<ModuleDescriptor, AbsolutePath> generatedProjectsByModuleDescriptor, Dictionary<AbsolutePath, ModuleDescriptor> generatedProjectsByPath, MultiValueDictionary<string, ModuleDescriptor> generatedProjectsByModuleName)
+        public NugetGenerationResult(
+            Dictionary<ModuleDescriptor, AbsolutePath> generatedProjectsByModuleDescriptor, 
+            Dictionary<AbsolutePath, ModuleDescriptor> generatedProjectsByPath, 
+            MultiValueDictionary<string, ModuleDescriptor> generatedProjectsByModuleName,
+            Dictionary<string, NugetAnalyzedPackage> nugetPackagesByModuleName)
         {
             GeneratedProjectsByModuleDescriptor = generatedProjectsByModuleDescriptor;
             GeneratedProjectsByPath = generatedProjectsByPath;
             GeneratedProjectsByModuleName = generatedProjectsByModuleName;
+            NugetPackagesByModuleName = nugetPackagesByModuleName;
         }
 
         /// <summary>
@@ -1751,5 +1768,14 @@ namespace BuildXL.FrontEnd.Nuget
         /// All package descriptors, indexed by name.
         /// </summary>
         public MultiValueDictionary<string, ModuleDescriptor> GeneratedProjectsByModuleName { get; set; }
+
+        public Dictionary<string, NugetAnalyzedPackage> NugetPackagesByModuleName { get; }
+
+        internal string GetOriginalNugetPackageName(ModuleDefinition def)
+        {
+            return NugetPackagesByModuleName != null && NugetPackagesByModuleName.TryGetValue(def.Descriptor.Name, out var value)
+                ? value.NugetName
+                : def.Descriptor.Name;
+        }
     }
 }

Public/Src/FrontEnd/Script/Tracing/Log.cs

@@ -414,6 +414,24 @@ namespace BuildXL.FrontEnd.Script.Tracing
             Message = EventConstants.LabeledProvenancePrefix + "Selector '{selector}' cannot be applied to receiver '{receiver}' in expression '{receiver}.{selector}' because receiver is of type 'any'. Values with type 'any' cannot be inspected in {ShortScriptName}.",
             Keywords = (int)(Keywords.UserMessage | Keywords.UserError))]
         public abstract void ReportPropertyAccessOnValueWithTypeAny(LoggingContext loggingContext, Location location, string receiver, string selector);
+
+        [GeneratedEvent(
+            (ushort)LogEventId.CGManifestValidationException,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Error,
+            EventTask = (ushort)Tasks.Parser,
+            Message = "{mesage}",
+            Keywords = (int)(Keywords.UserMessage | Keywords.UserError))]
+        public abstract void ReportComponentGovernanceValidationError(LoggingContext loggingContext, string mesage);
+
+        [GeneratedEvent(
+            (ushort)LogEventId.CGManifestGenerationException,
+            EventGenerators = EventGenerators.LocalOnly,
+            EventLevel = Level.Error,
+            EventTask = (ushort)Tasks.Parser,
+            Message = "{mesage}",
+            Keywords = (int)(Keywords.UserMessage | Keywords.UserError))]
+        public abstract void ReportComponentGovernanceGenerationError(LoggingContext loggingContext, string mesage);
     }
 
     /// <summary>

Public/Src/FrontEnd/Script/Tracing/LogEventId.cs

@@ -342,6 +342,9 @@ namespace BuildXL.FrontEnd.Script.Tracing
         ReportXmlUnsuportedTypeForSerialization = 9412,
         ReportUnsupportedTypeValueObjectException = 9413,
         DirectoryNotSupportedException = 9414,
+
+        CGManifestValidationException = 9415,
+        CGManifestGenerationException = 9416,
         // Obsolete syntax rules (starting from 9500)
 
         // Don't go beyond 9899

Public/Src/FrontEnd/UnitTests/Nuget/NugetCgManifestGeneratorTests.cs

@@ -0,0 +1,227 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using BuildXL.FrontEnd.Nuget;
+using BuildXL.FrontEnd.Sdk;
+using BuildXL.FrontEnd.Sdk.Mutable;
+using BuildXL.Utilities;
+using BuildXL.Utilities.Collections;
+using Newtonsoft.Json;
+using Test.BuildXL.TestUtilities.Xunit;
+using Xunit;
+
+namespace Test.BuildXL.FrontEnd.Nuget
+{
+    public sealed class NugetCgManifestGeneratorTests  : TemporaryStorageTestBase
+    {
+        private readonly FrontEndContext m_context;
+        private readonly NugetCgManifestGenerator m_generator;
+
+        public NugetCgManifestGeneratorTests()
+        {
+            m_context = FrontEndContext.CreateInstanceForTesting();
+            m_generator = new NugetCgManifestGenerator(m_context);
+        }
+
+        [Fact]
+        public void TestCompareForEquality()
+        {
+            string intendedManifest = @"{
+  ""Version"": 1,
+  ""Registrations"": [
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""Antlr4.Runtime.Standard"",
+          ""Version"": ""4.7.2""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""Aria.Cpp.SDK"",
+          ""Version"": ""8.5.6""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""ArtifactServices.App.Shared"",
+          ""Version"": ""17.150.28901-buildid9382555""
+        }
+      }
+    }
+  ]
+}
+";
+            string noSpaceManifest = @"{
+""Version"":1,""Registrations"":[{
+""Component"":{
+""Type"":""NuGet"",
+""NuGet"":{
+""Name"":""Antlr4.Runtime.Standard"",
+""Version"":""4.7.2""
+}}},
+{
+""Component"":{
+""Type"":""NuGet"",
+""NuGet"":{
+""Name"":""Aria.Cpp.SDK"",
+""Version"":""8.5.6""
+}}},
+{
+""Component"":{
+""Type"":""NuGet"",
+""NuGet"":{
+""Name"":""ArtifactServices.App.Shared"",
+""Version"":""17.150.28901-buildid9382555""
+}}}]}
+";
+            XAssert.IsTrue(m_generator.CompareForEquality(noSpaceManifest, intendedManifest));
+        }
+
+        [Fact]
+        public void TestCompareForEqualityInvalidFormat()
+        {
+            string validJson = "{ }";
+            string inValidJson = "{ ";
+            XAssert.IsFalse(m_generator.CompareForEquality(validJson, inValidJson));
+        }
+
+        private Package CreatePackage(string version)
+        {
+            AbsolutePath path = AbsolutePath.Create(m_context.PathTable, TemporaryDirectory + $"\\random.package.name\\{version}\\nu.spec");
+            var pathStr = path.ToString(m_context.PathTable);
+            var id = PackageId.Create(StringId.Create(m_context.StringTable, pathStr));
+            var desc = new PackageDescriptor();
+
+            return Package.Create(id, path, desc);
+
+        }
+
+        [Fact]
+        public void TestEmptyPackages()
+        {
+            MultiValueDictionary<string, Package> packages = new MultiValueDictionary<string, Package>();
+            var manifest = m_generator.GenerateCgManifestForPackages(packages);
+
+            var cgmanifest = new
+            {
+                Version = 1,
+                Registrations = new object[0]
+            };
+            string expectedManifest = JsonConvert.SerializeObject(cgmanifest, Formatting.Indented);
+
+            XAssert.IsTrue(m_generator.CompareForEquality(manifest, expectedManifest));
+        }
+
+        [Fact]
+        public void TestSinglePackage()
+        {
+            MultiValueDictionary<string, Package> packages = new MultiValueDictionary<string, Package>
+            {
+                { "test.package.name", CreatePackage("1.0.1") }
+            };
+
+            string expectedMainifest = @"
+{
+  ""Version"": 1,
+  ""Registrations"": [
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.name"",
+          ""Version"": ""1.0.1""
+        }
+      }
+    }
+  ]
+}";
+
+            XAssert.IsTrue(m_generator.CompareForEquality(expectedMainifest, m_generator.GenerateCgManifestForPackages(packages)));
+        }
+
+        [Fact]
+        public void TestSorted()
+        {
+            MultiValueDictionary<string, Package> packages = new MultiValueDictionary<string, Package>
+            {
+                { "test.package.name", CreatePackage("1.0.1") },
+                { "test.package.name", CreatePackage("1.0.2") },
+                { "test.package.name", CreatePackage("2.0.1") },
+                { "test.package.a", CreatePackage("5.1.1") },
+                { "test.package.z", CreatePackage("1.0.0") },
+                { "test.a.name", CreatePackage("10.0.1") }
+            };
+
+            string expectedMainifest = @"
+{
+  ""Version"": 1,
+  ""Registrations"": [
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.a.name"",
+          ""Version"": ""10.0.1""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.a"",
+          ""Version"": ""5.1.1""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.name"",
+          ""Version"": ""1.0.1""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.name"",
+          ""Version"": ""1.0.2""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.name"",
+          ""Version"": ""2.0.1""
+        }
+      }
+    },
+    {
+      ""Component"": {
+        ""Type"": ""NuGet"",
+        ""NuGet"": {
+          ""Name"": ""test.package.z"",
+          ""Version"": ""1.0.0""
+        }
+      }
+    }
+  ]
+}";
+
+            XAssert.IsTrue(m_generator.CompareForEquality(expectedMainifest, m_generator.GenerateCgManifestForPackages(packages)));
+        }
+    }
+}

Public/Src/FrontEnd/UnitTests/Nuget/Test.BuildXL.FrontEnd.Nuget.dsc

@@ -24,6 +24,7 @@ namespace Nuget {
             importFrom("BuildXL.Utilities").Configuration.dll,
             importFrom("BuildXL.Utilities").Script.Constants.dll,
             importFrom("BuildXL.Utilities.Instrumentation").Common.dll,
+            importFrom("Newtonsoft.Json").pkg,
             ...BuildXLSdk.tplPackages,
         ],
     });

Public/Src/Utilities/Configuration/IFrontEndConfiguration.cs

@@ -312,5 +312,21 @@ namespace BuildXL.Utilities.Configuration
         /// Whether or not the frontend is allowed to evaluate methods in the unsafe ambient.
         /// </summary>
         bool AllowUnsafeAmbient { get; }
+
+        /// <summary>
+        /// Generates a new cgmaiifest file and overwrites the existing cgmanifest file if it is outdated
+        /// </summary>
+        /// <remarks>
+        /// cgmanifest.json is used for Component Governance in CloudBuild
+        /// </remarks>
+        AbsolutePath GenerateCgManifestForNugets { get; }
+
+        /// <summary>
+        /// Validates the existing cgmaiifest file and throws error on mismatch
+        /// </summary>
+        /// <remarks>
+        /// cgmanifest.json is used for Component Governance in CloudBuild
+        /// </remarks>
+        AbsolutePath ValidateCgManifestForNugets { get; }
     }
 }

Public/Src/Utilities/Configuration/Mutable/FrontEndConfiguration.cs

@@ -69,6 +69,8 @@ namespace BuildXL.Utilities.Configuration.Mutable
             ReleaseWorkspaceBeforeEvaluation = template.ReleaseWorkspaceBeforeEvaluation;
             UnsafeOptimizedAstConversion = template.UnsafeOptimizedAstConversion;
             AllowUnsafeAmbient = template.AllowUnsafeAmbient;
+            GenerateCgManifestForNugets = template.GenerateCgManifestForNugets;
+            ValidateCgManifestForNugets = template.ValidateCgManifestForNugets;
         }
 
         /// <inheritdoc />
@@ -202,5 +204,11 @@ namespace BuildXL.Utilities.Configuration.Mutable
 
         /// <inheritdoc /> 
         public bool AllowUnsafeAmbient { get; set; }
+
+        /// <inheritdoc />
+        public AbsolutePath GenerateCgManifestForNugets { get; set; }
+
+        /// <inheritdoc />
+        public AbsolutePath ValidateCgManifestForNugets { get; set; }
     }
 }

Shared/Scripts/bxl.ps1

@@ -451,6 +451,9 @@ Log " version of BuildXL.";
 
 $AdditionalBuildXLArguments += "/environment:$($useDeployment.telemetryEnvironment)";
 
+$GenerateCgManifestFilePath = "$NormalizationDrive\cg\nuget\cgmanifest.json";
+# TODO (Rijul: Uncomment when changes in LKG) $AdditionalBuildXLArguments += "/generateCgManifestForNugest:$GenerateCgManifestFilePath";
+
 if (! $DoNotUseDefaultCacheConfigFilePath) {
 
     $cacheConfigPath = (Join-Path $cacheDirectory CacheCore.json);

cg/nuget/README.md

@@ -0,0 +1,2 @@
+cgmanifest.json is an auto-generated file. Any changes to this file will be discarded.
+Please add/remove NuGet Packages in config.dsc, the cgmanifest.json file will be updated automatically the next time you run bxl.