Add job to move sdks from prod to another storage account (#1499)

* Job to revert dev sdks to what is in prod

* renamed

* renamed various files and functions to be clearer and demonstrate functionality better

* added new sdk storage constant for sandbox account

* corrected storage account constant and updated copy script

* finalized logic to copy entire blob container to new storage account

* updated sdks job to have a dry run parameter, and to have actual parameters instead of variables

* made dry run default in case of empty val

* Pauld/sandbox to prod (#1501)

* Point python 3.6 get-pip to new source (#1498)

* add additional logic for python 3.6 download, pointing at another get-pip source

* fixed placement

* added pipeline that copies from a source storage account to the production one

* added parameters and dry run capabilities

* add clearer error message in case of bad argument

build/__sdkStorageConstants.sh

@@ -3,6 +3,7 @@
 ENABLE_DYNAMIC_INSTALL_KEY='ENABLE_DYNAMIC_INSTALL'
 SDK_STORAGE_BASE_URL_KEY_NAME='ORYX_SDK_STORAGE_BASE_URL'
 DEV_SDK_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
+SANDBOX_SDK_STORAGE_BASE_URL='https://oryxsdkssandbox.blob.core.windows.net'
 PROD_SDK_STORAGE_BASE_URL='https://oryxsdksprod.blob.core.windows.net'
 PROD_SDK_CDN_STORAGE_BASE_URL='https://oryx-cdn.microsoft.io'
 DEFAULT_VERSION_FILE_NAME='defaultVersion.txt'

build/constants.yaml

@@ -242,6 +242,7 @@
     enable-dynamic-install-key: ENABLE_DYNAMIC_INSTALL
     sdk-storage-base-url-key-name: ORYX_SDK_STORAGE_BASE_URL
     dev-sdk-storage-base-url: https://oryxsdksdev.blob.core.windows.net
+    sandbox-sdk-storage-base-url: https://oryxsdkssandbox.blob.core.windows.net
     prod-sdk-storage-base-url: https://oryxsdksprod.blob.core.windows.net
     prod-sdk-cdn-storage-base-url: https://oryx-cdn.microsoft.io
     default-version-file-name: defaultVersion.txt

images/__sdkStorageConstants.sh

@@ -3,6 +3,7 @@
 ENABLE_DYNAMIC_INSTALL_KEY='ENABLE_DYNAMIC_INSTALL'
 SDK_STORAGE_BASE_URL_KEY_NAME='ORYX_SDK_STORAGE_BASE_URL'
 DEV_SDK_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
+SANDBOX_SDK_STORAGE_BASE_URL='https://oryxsdkssandbox.blob.core.windows.net'
 PROD_SDK_STORAGE_BASE_URL='https://oryxsdksprod.blob.core.windows.net'
 PROD_SDK_CDN_STORAGE_BASE_URL='https://oryx-cdn.microsoft.io'
 DEFAULT_VERSION_FILE_NAME='defaultVersion.txt'

src/BuildScriptGenerator.Common/SdkStorageConstants.cs

@@ -7,6 +7,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Common
         public const string EnableDynamicInstallKey = "ENABLE_DYNAMIC_INSTALL";
         public const string SdkStorageBaseUrlKeyName = "ORYX_SDK_STORAGE_BASE_URL";
         public const string DevSdkStorageBaseUrl = "https://oryxsdksdev.blob.core.windows.net";
+        public const string SandboxSdkStorageBaseUrl = "https://oryxsdkssandbox.blob.core.windows.net";
         public const string ProdSdkStorageBaseUrl = "https://oryxsdksprod.blob.core.windows.net";
         public const string ProdSdkCdnStorageBaseUrl = "https://oryx-cdn.microsoft.io";
         public const string DefaultVersionFileName = "defaultVersion.txt";

src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go

@@ -5,6 +5,7 @@ package consts
 const EnableDynamicInstallKey string = "ENABLE_DYNAMIC_INSTALL"
 const SdkStorageBaseUrlKeyName string = "ORYX_SDK_STORAGE_BASE_URL"
 const DevSdkStorageBaseUrl string = "https://oryxsdksdev.blob.core.windows.net"
+const SandboxSdkStorageBaseUrl string = "https://oryxsdkssandbox.blob.core.windows.net"
 const ProdSdkStorageBaseUrl string = "https://oryxsdksprod.blob.core.windows.net"
 const ProdSdkCdnStorageBaseUrl string = "https://oryx-cdn.microsoft.io"
 const DefaultVersionFileName string = "defaultVersion.txt"

vsts/pipelines/copyAllBlobsToProd.yml

@@ -0,0 +1,53 @@
+variables:
+- group: Oryx
+  
+parameters:
+- name: sourceStorageAccount
+  displayName: Source Storage Account
+  type: string
+  default: https://oryxsdkssandbox.blob.core.windows.net
+  values:
+  - https://oryxsdkssandbox.blob.core.windows.net
+  - https://oryxsdksdev.blob.core.windows.net
+- name: dryRun
+  displayName: Dry Run?
+  type: boolean
+  default: true
+
+trigger: none
+
+jobs:
+    - job: CreateReleaseTag
+      pool:
+        name: AzurePipelines-EO
+        demands:
+          - ImageOverride -equals AzurePipelinesUbuntu20.04compliant
+      steps:
+      - checkout: self
+        clean: true
+
+      - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+        displayName: 'Component Detection - OSS Compliance'
+        inputs:
+          ignoreDirectories: '$(Build.SourcesDirectory)/tests'
+
+      - task: UseDotNet@2
+        displayName: 'Use .NET Core sdk 3.1.x'
+        inputs:
+          version: 3.1.x
+
+      - task: ShellScript@2
+        displayName: 'Copy all blobs from a source storage account to the prod storage account'
+        env:
+          DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN)
+          SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN)
+          PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN)
+        inputs:
+          scriptPath: ./vsts/scripts/copyAllBlobsToProdStorageAccount.sh
+          args:  ${{ parameters.sourceStorageAccount }} ${{ parameters.dryRun }}
+      
+      - task: ShellScript@2
+        displayName: 'Test Prod storage account'
+        inputs:
+          scriptPath: ./build/testIntegration.sh
+          args: StorageAccountTests=Prod

vsts/pipelines/copySdksFromProdToStorageAccount.yml

@@ -0,0 +1,53 @@
+variables:
+- group: Oryx
+
+parameters:
+- name: destinationStorageAccount
+  displayName: Destination Storage Account
+  type: string
+  default: https://oryxsdkssandbox.blob.core.windows.net
+  values:
+  - https://oryxsdkssandbox.blob.core.windows.net
+  - https://oryxsdksdev.blob.core.windows.net
+- name: dryRun
+  displayName: Dry Run?
+  type: boolean
+  default: true
+
+trigger: none
+
+jobs:
+    - job: CreateReleaseTag
+      pool:
+        name: AzurePipelines-EO
+        demands:
+          - ImageOverride -equals AzurePipelinesUbuntu20.04compliant
+      steps:
+      - checkout: self
+        clean: true
+
+      - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
+        displayName: 'Component Detection - OSS Compliance'
+        inputs:
+          ignoreDirectories: '$(Build.SourcesDirectory)/tests'
+
+      - task: UseDotNet@2
+        displayName: 'Use .NET Core sdk 3.1.x'
+        inputs:
+          version: 3.1.x
+
+      - task: ShellScript@2
+        displayName: 'Copy SDKs from the prod storage account to a destination storage account'
+        env:
+          DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN)
+          SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN)
+          PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN)
+        inputs:
+          scriptPath: ./vsts/scripts/copySdksFromProdToStorageAccount.sh
+          args:  ${{ parameters.destinationStorageAccount }} ${{ parameters.dryRun }}
+      
+      - task: ShellScript@2
+        displayName: 'Test Prod storage account'
+        inputs:
+          scriptPath: ./build/testIntegration.sh
+          args: StorageAccountTests=Prod

vsts/scripts/copyAllBlobsToProdStorageAccount.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT license.
+# --------------------------------------------------------------------------------------------
+
+set -ex
+
+declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && cd .. && pwd )
+source $REPO_DIR/platforms/__common.sh
+
+azCopyDir="/tmp/azcopy-tool"
+
+function blobContainerExistsInProd() {
+	local containerName="$1"
+	local exitCode=1
+	curl -I "$PROD_SDK_STORAGE_BASE_URL/$containerName?restype=container" 2> /tmp/curlError.txt 1> /tmp/curlOut.txt
+	grep "HTTP/1.1 200 OK" /tmp/curlOut.txt &> /dev/null
+	exitCode=$?
+	rm -f /tmp/curlOut.txt
+	rm -f /tmp/curlError.txt
+	if [ $exitCode -eq 0 ]; then
+		return 0
+	else
+        return 1
+	fi
+}
+
+function copyBlobContainerToProd() {
+    local platformName="$1"
+
+    if shouldOverwriteSdk || shouldOverwritePlatformSdk $platformName; then
+        echo
+        echo "Overwriting blob container '$platformName' in storage account '$destinationSdk'."
+        # azcopy copy [source] [destination] [flags]
+        if [ $dryRun == "false" ] ; then
+            "$azCopyDir/azcopy" copy \
+                "$sourceSdk/$platformName$sasToken" \ 
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" --overwrite true --recursive
+        else
+            "$azCopyDir/azcopy" copy \
+                "$sourceSdk/$platformName$sasToken" \ 
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" --overwrite true --recursive --dry-run
+        fi
+    elif blobContainerExistsInProd $platformName; then
+        echo
+        echo "Blob container '$platformName' already exists in Prod storage account. Skipping copying it..."
+    else
+        echo
+        echo "Blob container '$platformName' does not exist in Prod. Copying it from $sourceSdk..."
+        # azcopy copy [source] [destination] [flags]
+        if [ $dryRun == "false" ] ; then
+            "$azCopyDir/azcopy" copy \
+                "$sourceSdk/$platformName$sasToken" \ 
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" --overwrite false --recursive
+        else
+            "$azCopyDir/azcopy" copy \
+                "$sourceSdk/$platformName$sasToken" \ 
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" --overwrite false --recursive --dry-run
+        fi
+    fi
+}
+
+if [ ! -f "$azCopyDir/azcopy" ]; then
+    curl -SL https://aka.ms/downloadazcopy-v10-linux -o /tmp/azcopy_download.tar.gz
+    tar -xvf /tmp/azcopy_download.tar.gz -C /tmp
+    rm -rf /tmp/azcopy_download.tar.gz
+    mkdir -p $azCopyDir
+    cp /tmp/azcopy_linux_amd64_*/azcopy $azCopyDir
+
+    echo "Version of azcopy tool being used:"
+    $azCopyDir/azcopy --version
+fi
+
+sourceSdk=""
+sasToken=""
+
+if [ "$1" = $SANDBOX_SDK_STORAGE_BASE_URL ]; then
+    sourceSdk=$SANDBOX_SDK_STORAGE_BASE_URL
+    sasToken=$SANDBOX_STORAGE_SAS_TOKEN
+elif [ "$1" = $DEV_SDK_STORAGE_BASE_URL ]; then
+    sourceSdk=$DEV_SDK_STORAGE_BASE_URL
+    sasToken=$DEV_STORAGE_SAS_TOKEN
+else
+	echo "Error: $1 is an invalid source storage account url."
+	exit 1
+fi
+
+dryRun=$2
+if [ $dryRun != "true" ] && [ $dryRun != "false" ]; then
+	echo "Error: Dry run must be true or false. Was: '$dryRun'"
+	exit 1
+fi
+
+copyBlobContainerToProd "dotnet"
+copyBlobContainerToProd "python"
+copyBlobContainerToProd "nodejs"
+copyBlobContainerToProd "php"
+copyBlobContainerToProd "php-composer"
+copyBlobContainerToProd "ruby"
+copyBlobContainerToProd "java"
+copyBlobContainerToProd "maven"
+copyBlobContainerToProd "golang"

vsts/scripts/copySdksFromProdToStorageAccount.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT license.
+# --------------------------------------------------------------------------------------------
+
+set -ex
+
+declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && cd .. && pwd )
+source $REPO_DIR/platforms/__common.sh
+
+azCopyDir="/tmp/azcopy-tool"
+
+function blobContainerExistsInDestination() {
+	local containerName="$1"
+	local exitCode=1
+	curl -I "$destinationSdk/$containerName?restype=container" 2> /tmp/curlError.txt 1> /tmp/curlOut.txt
+	grep "HTTP/1.1 200 OK" /tmp/curlOut.txt &> /dev/null
+	exitCode=$?
+	rm -f /tmp/curlOut.txt
+	rm -f /tmp/curlError.txt
+	if [ $exitCode -eq 0 ]; then
+		return 0
+	else
+        return 1
+	fi
+}
+
+function copyBlobContainerFromProdToDestination() {
+    local platformName="$1"
+
+    if shouldOverwriteSdk || shouldOverwritePlatformSdk $platformName; then
+        echo
+        echo "Overwriting blob container '$platformName' in storage account '$destinationSdk'."
+        # azcopy copy [source] [destination] [flags]
+        if [ $dryRun == "false" ] ; then
+            "$azCopyDir/azcopy" copy \
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" \
+                "$destinationSdk/$platformName$sasToken" --overwrite true --recursive
+        else
+            "$azCopyDir/azcopy" copy \
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" \
+                "$destinationSdk/$platformName$sasToken" --overwrite true --recursive --dry-run
+        fi
+    elif blobContainerExistsInDestination $platformName; then
+        echo
+        echo "Blob container '$platformName' already exists in storage account '$destinationSdk'. Skipping copying it..."
+    else
+        echo
+        echo "Blob container '$platformName' does not exist in storage account '$destinationSdk'. Copying it from $PROD_SDK_STORAGE_BASE_URL..."
+        # azcopy copy [source] [destination] [flags]
+        if [ $dryRun == "false" ] ; then
+            "$azCopyDir/azcopy" copy \
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" \
+                "$destinationSdk/$platformName$sasToken" --overwrite false --recursive
+        else
+            "$azCopyDir/azcopy" copy \
+                "$PROD_SDK_STORAGE_BASE_URL/$platformName$PROD_STORAGE_SAS_TOKEN" \
+                "$destinationSdk/$platformName$sasToken" --overwrite false --recursive --dry-run
+        fi
+    fi
+}
+
+if [ ! -f "$azCopyDir/azcopy" ]; then
+    curl -SL https://aka.ms/downloadazcopy-v10-linux -o /tmp/azcopy_download.tar.gz
+    tar -xvf /tmp/azcopy_download.tar.gz -C /tmp
+    rm -rf /tmp/azcopy_download.tar.gz
+    mkdir -p $azCopyDir
+    cp /tmp/azcopy_linux_amd64_*/azcopy $azCopyDir
+
+    echo "Version of azcopy tool being used:"
+    $azCopyDir/azcopy --version
+fi
+
+destinationSdk=""
+sasToken=""
+
+if [ "$1" = $SANDBOX_SDK_STORAGE_BASE_URL ]; then
+    destinationSdk=$SANDBOX_SDK_STORAGE_BASE_URL
+    sasToken=$SANDBOX_STORAGE_SAS_TOKEN
+elif [ "$1" = $DEV_SDK_STORAGE_BASE_URL ]; then
+    destinationSdk=$DEV_SDK_STORAGE_BASE_URL
+    sasToken=$DEV_STORAGE_SAS_TOKEN
+else
+	echo "Error: $1 is an invalid destination storage account url."
+	exit 1
+fi
+
+dryRun=$2
+if [ $dryRun != "true" ] && [ $dryRun != "false" ]; then
+	echo "Error: Dry run must be true or false. Was: '$dryRun'"
+	exit 1
+fi
+
+copyBlobContainerFromProdToDestination "dotnet"
+copyBlobContainerFromProdToDestination "python"
+copyBlobContainerFromProdToDestination "nodejs"
+copyBlobContainerFromProdToDestination "php"
+copyBlobContainerFromProdToDestination "php-composer"
+copyBlobContainerFromProdToDestination "ruby"
+copyBlobContainerFromProdToDestination "java"
+copyBlobContainerFromProdToDestination "maven"
+copyBlobContainerFromProdToDestination "golang"