From 0f8eebe5079e533519dda3986431862643bc8153 Mon Sep 17 00:00:00 2001 From: snehapar9 <108305436+snehapar9@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:40:30 -0700 Subject: [PATCH] Concatenated blobs from multiple pages (#1545) * Concatenated blobs from multiple pages * Propogaated constants * Addressed PR comments * Replaced hardcoded "dotnet" by DotNetCoreConstants.PlatformName * Added helper to list all blobs * Fixed URL --- build/__sdkStorageConstants.sh | 2 +- build/constants.yaml | 2 +- images/__sdkStorageConstants.sh | 2 +- .../SdkStorageConstants.cs | 2 +- .../DotNetCoreSdkStorageVersionProvider.cs | 6 +-- .../Helpers/ListBlobsHelper.cs | 37 +++++++++++++++++++ .../SdkStorageVersionProviderBase.cs | 6 +-- .../common/consts/sdk_storage_constants.go | 2 +- .../StorageAccountSanityTestBase.cs | 5 +-- 9 files changed, 46 insertions(+), 18 deletions(-) create mode 100644 src/BuildScriptGenerator/Helpers/ListBlobsHelper.cs diff --git a/build/__sdkStorageConstants.sh b/build/__sdkStorageConstants.sh index a6c304456..56dc98484 100755 --- a/build/__sdkStorageConstants.sh +++ b/build/__sdkStorageConstants.sh @@ -11,7 +11,7 @@ DEFAULT_VERSION_FILE_NAME='defaultVersion.txt' DEFAULT_VERSION_FILE_PREFIX='defaultVersion' DEFAULT_VERSION_FILE_TYPE='txt' VERSIONS_TO_BUILD_FILE_NAME='versionsToBuild.txt' -CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata' +CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata&marker={2}' SDK_DOWNLOAD_SENTINEL_FILE_NAME='.oryx-sdkdownload-sentinel' SDK_VERSION_METADATA_NAME='Sdk_version' LEGACY_SDK_VERSION_METADATA_NAME='Version' diff --git a/build/constants.yaml b/build/constants.yaml index 439500abc..1e99947e2 100644 --- a/build/constants.yaml +++ b/build/constants.yaml @@ -291,7 +291,7 @@ default-version-file-prefix: defaultVersion default-version-file-type: txt versions-to-build-file-name: versionsToBuild.txt - container-metadata-url-format: '{0}/{1}?restype=container&comp=list&include=metadata' + container-metadata-url-format: '{0}/{1}?restype=container&comp=list&include=metadata&marker={2}' sdk-download-sentinel-file-name: .oryx-sdkdownload-sentinel # The following metadata names control the field that is uploaded to sdk storage alongside each sdk. # Legacy is used to provide backwards compatibility for images with an older CLI, as they depend on identifying diff --git a/images/__sdkStorageConstants.sh b/images/__sdkStorageConstants.sh index a6c304456..56dc98484 100644 --- a/images/__sdkStorageConstants.sh +++ b/images/__sdkStorageConstants.sh @@ -11,7 +11,7 @@ DEFAULT_VERSION_FILE_NAME='defaultVersion.txt' DEFAULT_VERSION_FILE_PREFIX='defaultVersion' DEFAULT_VERSION_FILE_TYPE='txt' VERSIONS_TO_BUILD_FILE_NAME='versionsToBuild.txt' -CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata' +CONTAINER_METADATA_URL_FORMAT='{0}/{1}?restype=container&comp=list&include=metadata&marker={2}' SDK_DOWNLOAD_SENTINEL_FILE_NAME='.oryx-sdkdownload-sentinel' SDK_VERSION_METADATA_NAME='Sdk_version' LEGACY_SDK_VERSION_METADATA_NAME='Version' diff --git a/src/BuildScriptGenerator.Common/SdkStorageConstants.cs b/src/BuildScriptGenerator.Common/SdkStorageConstants.cs index b52233baa..a61c855d8 100644 --- a/src/BuildScriptGenerator.Common/SdkStorageConstants.cs +++ b/src/BuildScriptGenerator.Common/SdkStorageConstants.cs @@ -15,7 +15,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Common public const string DefaultVersionFilePrefix = "defaultVersion"; public const string DefaultVersionFileType = "txt"; public const string VersionsToBuildFileName = "versionsToBuild.txt"; - public const string ContainerMetadataUrlFormat = "{0}/{1}?restype=container&comp=list&include=metadata"; + public const string ContainerMetadataUrlFormat = "{0}/{1}?restype=container&comp=list&include=metadata&marker={2}"; public const string SdkDownloadSentinelFileName = ".oryx-sdkdownload-sentinel"; public const string SdkVersionMetadataName = "Sdk_version"; public const string LegacySdkVersionMetadataName = "Version"; diff --git a/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreSdkStorageVersionProvider.cs b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreSdkStorageVersionProvider.cs index 0791c2d03..7b95480c8 100644 --- a/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreSdkStorageVersionProvider.cs +++ b/src/BuildScriptGenerator/DotNetCore/VersionProviders/DotNetCoreSdkStorageVersionProvider.cs @@ -57,11 +57,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.DotNetCore { var httpClient = this.HttpClientFactory.CreateClient("general"); var sdkStorageBaseUrl = this.GetPlatformBinariesStorageBaseUrl(); - var blobList = httpClient - .GetStringAsync($"{sdkStorageBaseUrl}/dotnet?restype=container&comp=list&include=metadata") - .Result; - - var xdoc = XDocument.Parse(blobList); + var xdoc = ListBlobsHelper.GetAllBlobs(sdkStorageBaseUrl, DotNetCoreConstants.PlatformName, httpClient); // keys represent runtime version, values represent sdk version var supportedVersions = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/BuildScriptGenerator/Helpers/ListBlobsHelper.cs b/src/BuildScriptGenerator/Helpers/ListBlobsHelper.cs new file mode 100644 index 000000000..f01478169 --- /dev/null +++ b/src/BuildScriptGenerator/Helpers/ListBlobsHelper.cs @@ -0,0 +1,37 @@ +// -------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. +// -------------------------------------------------------------------------------------------- +using System.Linq; +using System.Xml.Linq; +using Microsoft.Oryx.BuildScriptGenerator.Common; +using Microsoft.Oryx.BuildScriptGenerator.DotNetCore; + +namespace Microsoft.Oryx.BuildScriptGenerator +{ + internal static class ListBlobsHelper + { + public static XDocument GetAllBlobs(string sdkStorageBaseUrl, string platform, System.Net.Http.HttpClient httpClient) + { + var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platform, string.Empty); + var blobList = httpClient + .GetStringAsync(url) + .Result; + var xdoc = XDocument.Parse(blobList); + var marker = xdoc.Root.Element("NextMarker").Value; + + // if element's value is not empty, we iterate through every page by appending marker value to the url + // and consolidate blobs from all the pages. + do + { + url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platform, marker); + var blobListFromNextMarker = httpClient.GetStringAsync(url).Result; + var xdocFromNextMarker = XDocument.Parse(blobListFromNextMarker); + marker = xdocFromNextMarker.Root.Element("NextMarker").Value; + xdoc.Descendants("Blobs").LastOrDefault().AddAfterSelf(xdocFromNextMarker.Descendants("Blobs")); + } + while (!string.IsNullOrEmpty(marker)); + return xdoc; + } + } +} \ No newline at end of file diff --git a/src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs b/src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs index bdaa6e391..bc0341514 100644 --- a/src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs +++ b/src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs @@ -50,11 +50,8 @@ namespace Microsoft.Oryx.BuildScriptGenerator { this.logger.LogDebug("Getting list of available versions for platform {platformName}.", platformName); var httpClient = this.HttpClientFactory.CreateClient("general"); - var sdkStorageBaseUrl = this.GetPlatformBinariesStorageBaseUrl(); - var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, sdkStorageBaseUrl, platformName); - var blobList = httpClient.GetStringAsync(url).Result; - var xdoc = XDocument.Parse(blobList); + var xdoc = ListBlobsHelper.GetAllBlobs(sdkStorageBaseUrl, platformName, httpClient); var supportedVersions = new List(); var isStretch = string.Equals(this.commonOptions.DebianFlavor, OsTypes.DebianStretch, StringComparison.OrdinalIgnoreCase); @@ -62,7 +59,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator var sdkVersionMetadataName = isStretch ? SdkStorageConstants.LegacySdkVersionMetadataName : SdkStorageConstants.SdkVersionMetadataName; - foreach (var metadataElement in xdoc.XPathSelectElements($"//Blobs/Blob/Metadata")) { var childElements = metadataElement.Elements(); diff --git a/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go b/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go index d4958c4da..3bb00fcde 100644 --- a/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go +++ b/src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go @@ -13,7 +13,7 @@ const DefaultVersionFileName string = "defaultVersion.txt" const DefaultVersionFilePrefix string = "defaultVersion" const DefaultVersionFileType string = "txt" const VersionsToBuildFileName string = "versionsToBuild.txt" -const ContainerMetadataUrlFormat string = "{0}/{1}?restype=container&comp=list&include=metadata" +const ContainerMetadataUrlFormat string = "{0}/{1}?restype=container&comp=list&include=metadata&marker={2}" const SdkDownloadSentinelFileName string = ".oryx-sdkdownload-sentinel" const SdkVersionMetadataName string = "Sdk_version" const LegacySdkVersionMetadataName string = "Version" diff --git a/tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs b/tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs index 0eb4a95c6..a52e2117d 100644 --- a/tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs +++ b/tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs @@ -12,6 +12,7 @@ using System.Text.RegularExpressions; using System.Xml.Linq; using System.Xml.XPath; using Microsoft.Oryx.BuildScriptGenerator.Common; +using Microsoft.Oryx.BuildScriptGenerator; using Microsoft.Oryx.Integration.Tests; using Microsoft.Oryx.Tests.Common; using Xunit; @@ -198,9 +199,7 @@ namespace Oryx.Integration.Tests private XDocument GetMetadata(string platformName) { - var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, _storageUrl, platformName); - var blobList = _httpClient.GetStringAsync(url).Result; - return XDocument.Parse(blobList); + return ListBlobsHelper.GetAllBlobs(_storageUrl, platformName, _httpClient); } private List GetVersionsFromContainer(string debianFlavor, string platformName)