Split the packages up for better downloads (#1660)

2021-03-16 07:32:06 +02:00 · 2021-03-16 07:32:06 +02:00 · 0c634c273e
--- a/build.cake
+++ b/build.cake
@ -3,7 +3,7 @@
 #addin nuget:?package=Cake.FileHelpers&version=3.2.1
 #addin nuget:?package=Cake.Json&version=4.0.0
 #addin nuget:?package=SharpCompress&version=0.24.0
-#addin nuget:?package=Mono.ApiTools.NuGetDiff&version=1.3.0&loaddependencies=true
+#addin nuget:?package=Mono.ApiTools.NuGetDiff&version=1.3.2&loaddependencies=true
 #addin nuget:?package=Xamarin.Nuget.Validator&version=1.1.1

 #tool nuget:?package=mdoc&version=5.7.4.10
@ -29,6 +29,7 @@ DirectoryPath ROOT_PATH = MakeAbsolute(Directory("."));

 var SKIP_EXTERNALS = Argument ("skipexternals", "")
    .ToLower ().Split (new [] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+var SKIP_BUILD = Argument ("skipbuild", false);
 var PACK_ALL_PLATFORMS = Argument ("packall", Argument ("PackAllPlatforms", false));
 var BUILD_ALL_PLATFORMS = Argument ("buildall", Argument ("BuildAllPlatforms", false));
 var PRINT_ALL_ENV_VARS = Argument ("printAllEnvVars", false);
@ -54,7 +55,7 @@ var BUILD_NUMBER = EnvironmentVariable ("BUILD_NUMBER") ?? "0";
 var GIT_SHA = Argument ("gitSha", EnvironmentVariable ("GIT_SHA") ?? "");
 var GIT_BRANCH_NAME = Argument ("gitBranch", EnvironmentVariable ("GIT_BRANCH_NAME") ?? "");

-var PREVIEW_FEED_URL = "https://nugetized.blob.core.windows.net/skiasharp-eap/flatcontainer/{0}/{1}/{0}.{1}.nupkg"; // 0=id, 1=version
+var PREVIEW_FEED_URL = "https://pkgs.dev.azure.com/xamarin/public/_packaging/SkiaSharp/nuget/v3/index.json";

 var TRACKED_NUGETS = new Dictionary<string, Version> {
    { "SkiaSharp",                                     new Version (1, 57, 0) },
@ -108,6 +109,7 @@ Task ("externals")

 Task ("libs")
    .Description ("Build all managed assemblies.")
+    .WithCriteria(!SKIP_BUILD)
    .IsDependentOn ("externals")
    .Does (() =>
 {
@ -485,6 +487,11 @@ Task ("samples")
 ////////////////////////////////////////////////////////////////////////////////////////////////////

 Task ("nuget")
+    .Description ("Pack all NuGets.")
+    .IsDependentOn ("nuget-normal")
+    .IsDependentOn ("nuget-special");
+
+Task ("nuget-normal")
    .Description ("Pack all NuGets (build all required dependencies).")
    .IsDependentOn ("libs")
    .Does (() =>
@ -641,42 +648,91 @@ Task ("nuget")
            Information ("Metadata validation passed for: {0}", nupkgFile.GetFilename ());
        }
    }
+});

-    // special case for all the native assets
-    if (PACK_ALL_PLATFORMS)
-    {
-        EnsureDirectoryExists ($"{OUTPUT_SPECIAL_NUGETS_PATH}");
-        DeleteFiles ($"{OUTPUT_SPECIAL_NUGETS_PATH}/*.nupkg");
-        var specials = new Dictionary<string, string> {
-            { "_NativeAssets", "native" },
-            { "_NuGets", "nugets" },
-        };
-        foreach (var pair in specials) {
-            DeleteFiles ($"./output/{pair.Value}/*.nuspec");
+Task ("nuget-special")
+    .Description ("Pack all special NuGets.")
+    .IsDependentOn ("nuget-normal")
+    .Does (() =>
+{
+    EnsureDirectoryExists ($"{OUTPUT_SPECIAL_NUGETS_PATH}");
+    DeleteFiles ($"{OUTPUT_SPECIAL_NUGETS_PATH}/*.nupkg");

-            var nuspec = $"./output/{pair.Value}/{pair.Key}.nuspec";
+    // get a list of all the version number variants
+    var versions = new List<string> ();
+    if (!string.IsNullOrEmpty (PREVIEW_LABEL) && PREVIEW_LABEL.StartsWith ("pr.")) {
+        var v = $"0.0.0-{PREVIEW_LABEL}";
+        if (!string.IsNullOrEmpty (BUILD_NUMBER))
+            v += $".{BUILD_NUMBER}";
+        versions.Add (v);
+    } else {
+        if (!string.IsNullOrEmpty (GIT_SHA)) {
+            var v = $"0.0.0-commit.{GIT_SHA}";
+            if (!string.IsNullOrEmpty (BUILD_NUMBER))
+                v += $".{BUILD_NUMBER}";
+            versions.Add (v);
+        }
+        if (!string.IsNullOrEmpty (GIT_BRANCH_NAME)) {
+            var v = $"0.0.0-branch.{GIT_BRANCH_NAME.Replace ("/", ".")}";
+            if (!string.IsNullOrEmpty (BUILD_NUMBER))
+                v += $".{BUILD_NUMBER}";
+            versions.Add (v);
+        }
+    }

+    // get a list of all the nuspecs to pack
+    var specials = new Dictionary<string, string> ();
+
+    var nativePlatforms = GetDirectories ("./output/native/*")
+        .Select (d => d.GetDirectoryName ())
+        .ToArray ();
+    if (nativePlatforms.Length > 0) {
+        specials[$"_NativeAssets"] = $"native";
+        foreach (var platform in nativePlatforms) {
+            specials[$"_NativeAssets.{platform}"] = $"native/{platform}";
+        }
+    }
+    if (GetFiles ("./output/nugets/*.nupkg").Count > 0) {
+        specials[$"_NuGets"] = $"nugets";
+    }
+
+    foreach (var pair in specials) {
+        var id = pair.Key;
+        var path = pair.Value;
+        var nuspec = $"./output/{path}/{id}.nuspec";
+
+        DeleteFiles ($"./output/{path}/*.nuspec");
+
+        foreach (var packageVersion in versions) {
            // update the version
-            var xdoc = XDocument.Load ($"./nuget/{pair.Key}.nuspec");
+            var fn = id.StartsWith ("_NativeAssets.") ? "_NativeAssets" : id;
+            var xdoc = XDocument.Load ($"./nuget/{fn}.nuspec");
            var metadata = xdoc.Root.Element ("metadata");
-            var version = metadata.Element ("version");
+            metadata.Element ("version").Value = packageVersion;
+            metadata.Element ("id").Value = id;

-            if (!string.IsNullOrEmpty (PREVIEW_LABEL) && PREVIEW_LABEL.StartsWith ("pr.")) {
-                version.Value = "0.0.0-" + PREVIEW_LABEL;
-                xdoc.Save (nuspec);
-                PackageNuGet (nuspec, OUTPUT_SPECIAL_NUGETS_PATH, true);
-            } else {
-                version.Value = "0.0.0-commit." + GIT_SHA;
-                xdoc.Save (nuspec);
-                PackageNuGet (nuspec, OUTPUT_SPECIAL_NUGETS_PATH, true);
-
-                version.Value = "0.0.0-branch." + GIT_BRANCH_NAME.Replace ("/", ".");
-                xdoc.Save (nuspec);
-                PackageNuGet (nuspec, OUTPUT_SPECIAL_NUGETS_PATH, true);
+            if (id == "_NativeAssets") {
+                // handle the root package
+                var dependencies = metadata.Element ("dependencies");
+                foreach (var platform in nativePlatforms) {
+                    dependencies.Add (new XElement ("dependency",
+                        new XAttribute ("id", $"_NativeAssets.{platform}"),
+                        new XAttribute ("version", packageVersion)));
+                }
+            } else if (id.StartsWith ("_NativeAssets.")) {
+                // handle the dependencies
+                var platform = id.Substring (id.IndexOf (".") + 1);
+                var files = xdoc.Root.Element ("files");
+                files.Add (new XElement ("file",
+                    new XAttribute ("src", $"*/**"),
+                    new XAttribute ("target", $"tools/{platform}")));
            }

-            DeleteFiles ($"./output/{pair.Value}/*.nuspec");
+            xdoc.Save (nuspec);
+            PackageNuGet (nuspec, OUTPUT_SPECIAL_NUGETS_PATH, true);
        }
+
+        DeleteFiles ($"./output/{path}/*.nuspec");
    }
 });

--- a/cake/UpdateDocs.cake
+++ b/cake/UpdateDocs.cake
@ -38,19 +38,12 @@ void CopyChangelogs (DirectoryPath diffRoot, string id, string version)
 }

 Task ("docs-download-output")
-    .Does (() =>
+    .Does (async () =>
 {
    EnsureDirectoryExists ("./output");
    CleanDirectories ("./output");
-    EnsureDirectoryExists ("./output/temp");

-    var url = GetDownloadUrl ("_nugets");
-    DownloadFile (url, "./output/temp/nugets.nupkg");
-
-    Unzip ("./output/temp/nugets.nupkg", "./output/temp");
-    MoveDirectory ("./output/temp/tools", OUTPUT_NUGETS_PATH);
-
-    DeleteDirectory("./output/temp", new DeleteDirectorySettings { Recursive = true, Force = true });
+    await DownloadPackageAsync ("_nugets", OUTPUT_NUGETS_PATH);

    foreach (var id in TRACKED_NUGETS.Keys) {
        var version = GetVersion (id);
--- a/cake/UtilsManaged.cake
+++ b/cake/UtilsManaged.cake
@ -18,25 +18,6 @@ void PackageNuGet(FilePath nuspecPath, DirectoryPath outputPath, bool allowDefau
    NuGetPack(nuspecPath, settings);
 }

-void RunNuGetRestorePackagesConfig(FilePath sln)
-{
-    var dir = sln.GetDirectory();
-
-    var nugetSources = new [] { OUTPUT_NUGETS_PATH.FullPath, "https://api.nuget.org/v3/index.json" };
-
-    EnsureDirectoryExists(OUTPUT_NUGETS_PATH);
-
-    var settings = new NuGetRestoreSettings {
-        ToolPath = NuGetToolPath,
-        Source = nugetSources,
-        NoCache = true,
-        PackagesDirectory = dir.Combine("packages"),
-    };
-
-    foreach (var config in GetFiles(dir + "/**/packages.config"))
-        NuGetRestore(config, settings);
-}
-
 void RunTests(FilePath testAssembly, bool is32)
 {
    var dir = testAssembly.GetDirectory();
@ -104,7 +85,7 @@ void RunNetCorePublish(FilePath testProject, DirectoryPath output)
 void RunCodeCoverage(string testResultsGlob, DirectoryPath output)
 {
    try {
-        RunProcess ("reportgenerator", new ProcessSettings {
+        RunProcess("reportgenerator", new ProcessSettings {
            Arguments = 
                $"-reports:{testResultsGlob} " +
                $"-targetdir:{output} " +
@ -112,13 +93,13 @@ void RunCodeCoverage(string testResultsGlob, DirectoryPath output)
                $"-assemblyfilters:-*.Tests"
        });
    } catch (Exception ex) {
-        Error ("Make sure to install the 'dotnet-reportgenerator-globaltool' .NET Core global tool.");
-        Error (ex);
+        Error("Make sure to install the 'dotnet-reportgenerator-globaltool' .NET Core global tool.");
+        Error(ex);
        throw;
    }
    var xml = $"{output}/Cobertura.xml";
-    var root = FindRegexMatchGroupsInFile (xml, @"<source>(.*)<\/source>", 0)[1].Value;
-    ReplaceTextInFiles (xml, root, "");
+    var root = FindRegexMatchGroupsInFile(xml, @"<source>(.*)<\/source>", 0)[1].Value;
+    ReplaceTextInFiles(xml, root, "");
 }

 IEnumerable<(string Name, string Value)> CreateTraitsDictionary(string args)
@ -260,17 +241,59 @@ async Task<NuGetDiff> CreateNuGetDiffAsync()
    }
 }

-string GetDownloadUrl(string id)
+async Task DownloadPackageAsync(string id, DirectoryPath outputDirectory)
 {
    var version = "0.0.0-";
-    if (!string.IsNullOrEmpty (PREVIEW_LABEL) && PREVIEW_LABEL.StartsWith ("pr."))
-        version += PREVIEW_LABEL.ToLower ();
-    else if (!string.IsNullOrEmpty (GIT_SHA))
-        version += "commit." + GIT_SHA.ToLower ();
-    else if (!string.IsNullOrEmpty (GIT_BRANCH_NAME))
-        version += "branch." + GIT_BRANCH_NAME.Replace ("/", ".").ToLower ();
+    if (!string.IsNullOrEmpty(PREVIEW_LABEL) && PREVIEW_LABEL.StartsWith("pr."))
+        version += PREVIEW_LABEL.ToLower();
+    else if (!string.IsNullOrEmpty(GIT_SHA))
+        version += "commit." + GIT_SHA.ToLower();
+    else if (!string.IsNullOrEmpty(GIT_BRANCH_NAME))
+        version += "branch." + GIT_BRANCH_NAME.Replace("/", ".").ToLower();
    else
        version += "branch.main";
+    version += ".*";

-    return string.Format (PREVIEW_FEED_URL, id.ToLower(), version);
+    var filter = new NuGetVersions.Filter {
+        IncludePrerelease = true,
+        SourceUrl = PREVIEW_FEED_URL,
+        VersionRange = VersionRange.Parse(version),
+    };
+
+    var latestVersion = await NuGetVersions.GetLatestAsync(id, filter);
+
+    var comparer = new NuGetDiff(PREVIEW_FEED_URL);
+    comparer.PackageCache = PACKAGE_CACHE_PATH.FullPath;
+
+    await Download(id, latestVersion);
+
+    async Task Download(string currentId, NuGetVersion currentVersion)
+    {
+        currentId = currentId.ToLower();
+
+        Information($"Downloading: {currentId}...");
+
+        var root = await comparer.ExtractCachedPackageAsync(currentId, currentVersion);
+        var toolsDir = $"{root}/tools/";
+        if (DirectoryExists(toolsDir)) {
+            var allFiles = GetFiles(toolsDir + "**/*");
+            foreach (var file in allFiles) {
+                var relative = MakeAbsolute(Directory(toolsDir)).GetRelativePath(file);
+                var dir = $"{outputDirectory}/{relative.GetDirectory()}";
+                EnsureDirectoryExists(dir);
+                CopyFileToDirectory(file, dir);
+            }
+        }
+
+        var nuspec = $"{root}/{currentId}.nuspec";
+        var xdoc = XDocument.Load(nuspec);
+        var xmlns = xdoc.Root.Name.Namespace;
+        var dependencies = xdoc.Root.Descendants(xmlns + "dependency").ToArray();
+
+        foreach (var dep in dependencies) {
+            var depId = dep.Attribute("id").Value;
+            var depVersion = dep.Attribute("version").Value;
+            await Download(depId, NuGetVersion.Parse(depVersion));
+        }
+    }
 }
--- a/cake/externals.cake
+++ b/cake/externals.cake
@ -12,6 +12,7 @@ foreach (var cake in GetFiles("native/*/build.cake"))

    var task = Task($"externals-{native}")
        .WithCriteria(should)
+        .WithCriteria(!SKIP_BUILD)
        .Does(() => RunCake(localCake, "Default"));

    externalsTask.IsDependentOn(task);
@ -25,19 +26,12 @@ Task("externals-osx")
 ////////////////////////////////////////////////////////////////////////////////////////////////////

 Task("externals-download")
-    .Does(() =>
+    .Does(async () =>
 {
    EnsureDirectoryExists ("./output");
    CleanDirectories ("./output");
-    EnsureDirectoryExists ("./output/temp");

-    var url = GetDownloadUrl ("_nativeassets");
-    DownloadFile (url, "./output/temp/nativeassets.nupkg");
-
-    Unzip ("./output/temp/nativeassets.nupkg", "./output/temp");
-    MoveDirectory ("./output/temp/tools", "./output/native");
-
-    DeleteDirectory("./output/temp", new DeleteDirectorySettings { Recursive = true, Force = true });
+    await DownloadPackageAsync("_nativeassets", "./output/native");
 });

 ////////////////////////////////////////////////////////////////////////////////////////////////////
--- a/cake/msbuild.cake
+++ b/cake/msbuild.cake
@ -2,6 +2,29 @@ DirectoryPath PACKAGE_CACHE_PATH = MakeAbsolute(ROOT_PATH.Combine("externals/pac
 DirectoryPath OUTPUT_NUGETS_PATH = MakeAbsolute(ROOT_PATH.Combine("output/nugets"));
 DirectoryPath OUTPUT_SPECIAL_NUGETS_PATH = MakeAbsolute(ROOT_PATH.Combine("output/special-nugets"));

+var NUGETS_SOURCES = new [] {
+    OUTPUT_NUGETS_PATH.FullPath,
+    "https://api.nuget.org/v3/index.json",
+    "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
+};
+
+void RunNuGetRestorePackagesConfig(FilePath sln)
+{
+    var dir = sln.GetDirectory();
+
+    EnsureDirectoryExists(OUTPUT_NUGETS_PATH);
+
+    var settings = new NuGetRestoreSettings {
+        ToolPath = NuGetToolPath,
+        Source = NUGETS_SOURCES,
+        NoCache = true,
+        PackagesDirectory = dir.Combine("packages"),
+    };
+
+    foreach (var config in GetFiles(dir + "/**/packages.config"))
+        NuGetRestore(config, settings);
+}
+
 void RunMSBuild(
    FilePath solution,
    string platform = "Any CPU",
@ -12,12 +35,6 @@ void RunMSBuild(
    string configuration = null,
    Dictionary<string, string> properties = null)
 {
-    var nugetSources = new [] {
-        OUTPUT_NUGETS_PATH.FullPath,
-        "https://api.nuget.org/v3/index.json",
-        "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
-    };
-
    EnsureDirectoryExists(OUTPUT_NUGETS_PATH);

    MSBuild(solution, c => {
@ -68,8 +85,8 @@ void RunMSBuild(
                c.Properties [prop.Key] = new [] { prop.Value };
            }
        }
-        // c.Properties ["RestoreSources"] = nugetSources;
+        // c.Properties ["RestoreSources"] = NUGETS_SOURCES;
        var sep = IsRunningOnWindows() ? ";" : "%3B";
-        c.ArgumentCustomization = args => args.Append($"/p:RestoreSources=\"{string.Join(sep, nugetSources)}\"");
+        c.ArgumentCustomization = args => args.Append($"/p:RestoreSources=\"{string.Join(sep, NUGETS_SOURCES)}\"");
    });
 }
--- a/nuget/_NativeAssets.nuspec
+++ b/nuget/_NativeAssets.nuspec
@ -5,14 +5,18 @@
    <id>_NativeAssets</id>
    <title>Build Native Assets</title>
    <version>1.0.0</version>
-    <description>All the native assets from the build</description>
-    <summary>All the native assets from the build</summary>
+    <description>All the native assets from the build.</description>
+    <summary>All the native assets from the build.</summary>
    <authors>Microsoft</authors>

+    <dependencies>
+      <!-- <dependency id="_NativeAssets.windows" version="1.0.0" /> -->
+    </dependencies>
+
  </metadata>
  <files>

-    <file src="*/**" target="tools/" />
+    <!-- <file src="*/**" target="tools/" /> -->

  </files>
 </package>
--- a/nuget/_NuGets.nuspec
+++ b/nuget/_NuGets.nuspec
@ -5,8 +5,8 @@
    <id>_NuGets</id>
    <title>Build NuGets</title>
    <version>1.0.0</version>
-    <description>All the NuGet packages from the build</description>
-    <summary>All the NuGet packages from the build</summary>
+    <description>All the stable NuGet packages from the build.</description>
+    <summary>All the stable NuGet packages from the build.</summary>
    <authors>Microsoft</authors>

  </metadata>
--- a/scripts/azure-pipelines.yml
+++ b/scripts/azure-pipelines.yml
@ -322,7 +322,7 @@ stages:
          displayName: Managed (Windows)
          vmImage: $(VM_IMAGE_WINDOWS)
          target: libs
-          additionalArgs: --exclusive
+          additionalArgs: --skipExternals="all"
          requiredArtifacts:
            - native_android_x86_windows
            - native_android_x64_windows
@ -348,7 +348,7 @@ stages:
          displayName: Managed (macOS)
          vmImage: $(VM_IMAGE_MAC)
          target: libs
-          additionalArgs: --exclusive
+          additionalArgs: --skipExternals="all"
          requiredArtifacts:
            - native_android_x86_macos
            - native_android_x64_macos
@ -368,7 +368,7 @@ stages:
          vmImage: $(VM_IMAGE_LINUX)
          packages: $(MANAGED_LINUX_PACKAGES)
          target: libs
-          additionalArgs: --exclusive
+          additionalArgs: --skipExternals="all"
          requiredArtifacts:
            - native_linux_x64_linux
            - native_linux_arm_linux
@ -392,7 +392,7 @@ stages:
          displayName: Package NuGets
          vmImage: $(VM_IMAGE_WINDOWS)
          target: nuget
-          additionalArgs: --packall=true --exclusive
+          additionalArgs: --packall=true --skipbuild=true
          installWindowsSdk: false
          shouldPublish: true
          requiredArtifacts: