Concatenated blobs from multiple pages (#1545)

* Concatenated blobs from multiple pages

* Propogaated constants

* Addressed PR comments

* Replaced hardcoded "dotnet" by DotNetCoreConstants.PlatformName

* Added helper to list all blobs

* Fixed URL

build/__sdkStorageConstants.sh

@@ -11,7 +11,7 @@ DEFAULT_VERSION_FILE_NAME='defaultVersion.txt'
 DEFAULT_VERSION_FILE_PREFIX='defaultVersion'
 DEFAULT_VERSION_FILE_TYPE='txt'
 VERSIONS_TO_BUILD_FILE_NAME='versionsToBuild.txt'
-CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata'
+CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata&marker={2}'
 SDK_DOWNLOAD_SENTINEL_FILE_NAME='.oryx-sdkdownload-sentinel'
 SDK_VERSION_METADATA_NAME='Sdk_version'
 LEGACY_SDK_VERSION_METADATA_NAME='Version'

build/constants.yaml

@@ -291,7 +291,7 @@
     default-version-file-prefix: defaultVersion
     default-version-file-type: txt
     versions-to-build-file-name: versionsToBuild.txt
-    container-metadata-url-format: '{0}/{1}?restype=container&comp=list&include=metadata'
+    container-metadata-url-format: '{0}/{1}?restype=container&comp=list&include=metadata&marker={2}'
     sdk-download-sentinel-file-name: .oryx-sdkdownload-sentinel
     # The following metadata names control the field that is uploaded to sdk storage alongside each sdk.
     # Legacy is used to provide backwards compatibility for images with an older CLI, as they depend on identifying

images/__sdkStorageConstants.sh

@@ -11,7 +11,7 @@ DEFAULT_VERSION_FILE_NAME='defaultVersion.txt'
 DEFAULT_VERSION_FILE_PREFIX='defaultVersion'
 DEFAULT_VERSION_FILE_TYPE='txt'
 VERSIONS_TO_BUILD_FILE_NAME='versionsToBuild.txt'
-CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata'
+CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata&marker={2}'
 SDK_DOWNLOAD_SENTINEL_FILE_NAME='.oryx-sdkdownload-sentinel'
 SDK_VERSION_METADATA_NAME='Sdk_version'
 LEGACY_SDK_VERSION_METADATA_NAME='Version'

src/BuildScriptGenerator.Common/SdkStorageConstants.cs

@@ -15,7 +15,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Common
         public const string DefaultVersionFilePrefix = "defaultVersion";
         public const string DefaultVersionFileType = "txt";
         public const string VersionsToBuildFileName = "versionsToBuild.txt";
-        public const string ContainerMetadataUrlFormat = "{0}/{1}?restype=container&comp=list&include=metadata";
+        public const string ContainerMetadataUrlFormat = "{0}/{1}?restype=container&comp=list&include=metadata&marker={2}";
         public const string SdkDownloadSentinelFileName = ".oryx-sdkdownload-sentinel";
         public const string SdkVersionMetadataName = "Sdk_version";
         public const string LegacySdkVersionMetadataName = "Version";

src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreSdkStorageVersionProvider.cs

@@ -57,11 +57,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore
             {
                 var httpClient = this.HttpClientFactory.CreateClient("general");
                 var sdkStorageBaseUrl = this.GetPlatformBinariesStorageBaseUrl();
-                var blobList = httpClient
-                    .GetStringAsync($"{sdkStorageBaseUrl}/dotnet?restype=container&comp=list&include=metadata")
-                    .Result;
-
-                var xdoc = XDocument.Parse(blobList);
+                var xdoc = ListBlobsHelper.GetAllBlobs(sdkStorageBaseUrl, DotNetCoreConstants.PlatformName, httpClient);
 
                 // keys represent runtime version, values represent sdk version
                 var supportedVersions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);

src/BuildScriptGenerator/Helpers/ListBlobsHelper.cs

@@ -0,0 +1,37 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+using System.Linq;
+using System.Xml.Linq;
+using Microsoft.Oryx.BuildScriptGenerator.Common;
+using Microsoft.Oryx.BuildScriptGenerator.DotNetCore;
+
+namespace Microsoft.Oryx.BuildScriptGenerator
+{
+    internal static class ListBlobsHelper
+    {
+        public static XDocument GetAllBlobs(string sdkStorageBaseUrl, string platform, System.Net.Http.HttpClient httpClient)
+        {
+            var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platform, string.Empty);
+            var blobList = httpClient
+                .GetStringAsync(url)
+                .Result;
+            var xdoc = XDocument.Parse(blobList);
+            var marker = xdoc.Root.Element("NextMarker").Value;
+
+            // if <NextMarker> element's value is not empty, we iterate through every page by appending marker value to the url
+            // and consolidate blobs from all the pages.
+            do
+            {
+                url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platform, marker);
+                var blobListFromNextMarker = httpClient.GetStringAsync(url).Result;
+                var xdocFromNextMarker = XDocument.Parse(blobListFromNextMarker);
+                marker = xdocFromNextMarker.Root.Element("NextMarker").Value;
+                xdoc.Descendants("Blobs").LastOrDefault().AddAfterSelf(xdocFromNextMarker.Descendants("Blobs"));
+            }
+            while (!string.IsNullOrEmpty(marker));
+            return xdoc;
+        }
+    }
+}

src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs

@@ -50,11 +50,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         {
             this.logger.LogDebug("Getting list of available versions for platform {platformName}.", platformName);
             var httpClient = this.HttpClientFactory.CreateClient("general");
-
             var sdkStorageBaseUrl = this.GetPlatformBinariesStorageBaseUrl();
-            var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platformName);
-            var blobList = httpClient.GetStringAsync(url).Result;
-            var xdoc = XDocument.Parse(blobList);
+            var xdoc = ListBlobsHelper.GetAllBlobs(sdkStorageBaseUrl, platformName, httpClient);
             var supportedVersions = new List<string>();
 
             var isStretch = string.Equals(this.commonOptions.DebianFlavor, OsTypes.DebianStretch, StringComparison.OrdinalIgnoreCase);
@@ -62,7 +59,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator
             var sdkVersionMetadataName = isStretch
                 ? SdkStorageConstants.LegacySdkVersionMetadataName
                 : SdkStorageConstants.SdkVersionMetadataName;
-
             foreach (var metadataElement in xdoc.XPathSelectElements($"//Blobs/Blob/Metadata"))
             {
                 var childElements = metadataElement.Elements();

src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go

@@ -13,7 +13,7 @@ const DefaultVersionFileName string = "defaultVersion.txt"
 const DefaultVersionFilePrefix string = "defaultVersion"
 const DefaultVersionFileType string = "txt"
 const VersionsToBuildFileName string = "versionsToBuild.txt"
-const ContainerMetadataUrlFormat string = "{0}/{1}?restype=container&comp=list&include=metadata"
+const ContainerMetadataUrlFormat string = "{0}/{1}?restype=container&comp=list&include=metadata&marker={2}"
 const SdkDownloadSentinelFileName string = ".oryx-sdkdownload-sentinel"
 const SdkVersionMetadataName string = "Sdk_version"
 const LegacySdkVersionMetadataName string = "Version"

tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs

@@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
 using System.Xml.Linq;
 using System.Xml.XPath;
 using Microsoft.Oryx.BuildScriptGenerator.Common;
+using Microsoft.Oryx.BuildScriptGenerator;
 using Microsoft.Oryx.Integration.Tests;
 using Microsoft.Oryx.Tests.Common;
 using Xunit;
@@ -198,9 +199,7 @@ namespace Oryx.Integration.Tests
 
         private XDocument GetMetadata(string platformName)
         {
-            var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, _storageUrl, platformName);
-            var blobList = _httpClient.GetStringAsync(url).Result;
-            return XDocument.Parse(blobList);
+            return ListBlobsHelper.GetAllBlobs(_storageUrl, platformName, _httpClient);
         }
 
         private List<string> GetVersionsFromContainer(string debianFlavor, string platformName)