Add overwrite option when publish sdks from dev to prod (#1307)

* remove from py package example

* update py package version

* add overwrite options for publishing prod sdks

* address feedbacks

* overwrite when uploading files to dev storage account

* update __common.sh

* update docs

Co-authored-by: Stella Qian <zixuan.qian@microsoft.com>

platforms/README.md

@@ -1,4 +1,9 @@
 ## How to add a new release version and publish to Azure Blob Storage?
 Add a new version and its SHA, GPG keys information to `versionsToBuild.txt` file. Update version constants in `constants.yaml` and other files.
 If it's a default version, reset `defaultVersion.txt` file.
-Manually trigger the platform binary build pipeline after sent PR and update related tests.
+Manually trigger the platform binary build pipeline after sent PR and update related tests.
+
+# Options to overwrite platform binaries when publishing SDKs to dev or prod storage account
+We set up environment variable during the pipeline execution. When trigger a new build in Oryx-PlatformBinary-<PlatformName> in Azure DevOps pipelines, click the 'run pipeline'->'Advanced options'->'Variables' and add the variable name and value there.
+Set up OVERWRITE_EXISTING_SDKS="true" to overwrite SDKs in all platforms.
+Set up OVERWRITE_EXISTING_SDKS_<PlatformName>="true" to overwrite SDKs in a specific platform.

platforms/__common.sh

@@ -51,6 +51,75 @@ shouldOverwriteSdk() {
 	fi
 }
 
+shouldOverwritePlatformSdk() {
+	local platform="$1"
+	case $platform in
+        "php")
+            	if [ "$OVERWRITE_EXISTING_SDKS_PHP" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+	    	;;
+        "php-composer")
+            	if [ "$OVERWRITE_EXISTING_SDKS_PHP_COMPOSER" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+            	;;
+	"java")
+	    		if [ "$OVERWRITE_EXISTING_SDKS_JAVA" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+	    	;;
+        "maven")
+	    		if [ "$OVERWRITE_EXISTING_SDKS_MAVEN" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+	    	;;
+	"nodejs")
+	 		if [ "$OVERWRITE_EXISTING_SDKS_NODE" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+	    	;;
+        "ruby")
+		if [ "$OVERWRITE_EXISTING_SDKS_RUBY" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+		;;
+	"python")
+		if [ "$OVERWRITE_EXISTING_SDKS_PYTHON" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+		;;
+        "golang")
+		if [ "$OVERWRITE_EXISTING_SDKS_GOLANG" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+		;;
+	"dotnet")
+		if [ "$OVERWRITE_EXISTING_SDKS_DOTNETCORE" == "true" ]; then
+					return 0
+				else
+					return 1
+				fi
+		;;
+	esac
+}
+
 getSdkFromImage() {
 	local imageName="$1"
 	local hostVolumeDir="$2"

platforms/dotnet/getDotNetSdk.sh

@@ -28,7 +28,7 @@ getDotNetCoreSdk() {
 			downloadedFile=dotnet-$debianFlavor-$sdkVersion.tar.gz
 	fi
 
-	if shouldBuildSdk dotnet $downloadedFile || shouldOverwriteSdk || shouldOverwriteDotNetCoreSdk; then
+	if shouldBuildSdk dotnet $downloadedFile || shouldOverwriteSdk || shouldOverwritePlatformSdk dotnet; then
 		echo "Downloading .NET Core SDK version '$sdkVersion'..."
 		echo
 
@@ -57,14 +57,6 @@ getDotNetCoreSdk() {
 	fi
 }
 
-shouldOverwriteDotNetCoreSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_DOTNETCORE" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo
 echo "Getting .NET Core Sdks..."
 echo

platforms/golang/getGolangSdk.sh

@@ -29,7 +29,7 @@ getGolangSdk() {
 			downloadedFile=golang-$debianFlavor-$sdkVersion.tar.gz
 	fi
 
-	if shouldBuildSdk golang $downloadedFile || shouldOverwriteSdk || shouldOverwriteGolangSdk; then
+	if shouldBuildSdk golang $downloadedFile || shouldOverwriteSdk || shouldOverwritePlatformSdk golang; then
 		echo "Downloading golang SDK version '$sdkVersion'..."
 		echo
 
@@ -54,14 +54,6 @@ getGolangSdk() {
 	fi
 }
 
-shouldOverwriteGolangSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_GOLANG" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo
 echo "Getting golang Sdks..."
 echo

platforms/java/getJavaSdk.sh

@@ -62,7 +62,7 @@ downloadJavaSdk()
         return
     fi
 
-    if shouldBuildSdk java $tarFileName || shouldOverwriteSdk || shouldOverwriteJavaSdk; then
+    if shouldBuildSdk java $tarFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk java; then
         local baseUrl="https://github.com/AdoptOpenJDK/openjdk${majorVersion}-binaries/releases/download"
         if [ "$majorVersion" == "10" ] && [ "$JDK_BUILD_NUMBER" == "13" ]; then
             url="$baseUrl/jdk-10.0.2%2B13.1/OpenJDK10U-jdk_x64_linux_hotspot_10.0.2_13.tar.gz"
@@ -83,14 +83,6 @@ downloadJavaSdk()
     fi
 }
 
-shouldOverwriteNodeSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_JAVA" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Downloading Java SDK..."
 echo
 buildPlatform "$javaPlatformDir/versionsToBuild.txt" downloadJavaSdk

platforms/java/maven/getMaven.sh

@@ -26,7 +26,7 @@ downloadMavenBinary()
 			tarFileName=maven-$debianFlavor-$version.tar.gz
 	fi
 
-    if shouldBuildSdk maven $tarFileName || shouldOverwriteSdk || shouldOverwriteMavenBinary; then
+    if shouldBuildSdk maven $tarFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk maven; then
         curl -L "$baseUrl/$version/binaries/apache-maven-$version-bin.tar.gz" -o $tarFileName
         rm -rf extracted
         mkdir -p extracted
@@ -38,14 +38,6 @@ downloadMavenBinary()
     fi
 }
 
-shouldOverwriteMavenBinary() {
-	if [ "$OVERWRITE_EXISTING_SDKS_MAVEN" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Downloading Maven binary..."
 echo
 buildPlatform "$mavenPlatformDir/versionsToBuild.txt" downloadMavenBinary

platforms/nodejs/getNode.sh

@@ -28,7 +28,7 @@ getNode() {
 			tarFileName=nodejs-$debianFlavor-$version.tar.gz
 	fi
 
-	if shouldBuildSdk nodejs $tarFileName || shouldOverwriteSdk || shouldOverwriteNodeSdk; then
+	if shouldBuildSdk nodejs $tarFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk nodejs; then
 		echo "Getting Node version '$version'..."
 		echo
 
@@ -50,14 +50,6 @@ getNode() {
 	fi
 }
 
-shouldOverwriteNodeSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_NODE" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Getting Node Sdk..."
 echo
 buildPlatform "$nodePlatformDir/versionsToBuild.txt" getNode

platforms/php/buildPhp.sh

@@ -45,7 +45,7 @@ buildPhp() {
 
 	cp "$phpPlatformDir/defaultVersion.txt" "$targetDir"
 
-	if shouldBuildSdk php $phpSdkFileName || shouldOverwriteSdk || shouldOverwritePhpSdk; then
+	if shouldBuildSdk php $phpSdkFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk php; then
 		if ! $builtPhpPrereqs; then
 			buildPhpPrereqsImage
 		fi
@@ -84,7 +84,7 @@ buildPhpComposer() {
         composerSdkFileName=php-composer-$debianFlavor-$version.tar.gz
     fi
 
-	if shouldBuildSdk php-composer $composerSdkFileName || shouldOverwriteSdk || shouldOverwritePhpComposerSdk; then
+	if shouldBuildSdk php-composer $composerSdkFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk php-composer; then
 		if ! $builtPhpPrereqs; then
 			buildPhpPrereqsImage
 		fi
@@ -111,22 +111,6 @@ buildPhpComposer() {
 	fi
 }
 
-shouldOverwritePhpSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_PHP" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
-shouldOverwritePhpComposerSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_PHP_COMPOSER" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Building Php..."
 echo
 buildPlatform "$phpPlatformDir/versionsToBuild.txt" buildPhp

platforms/python/buildPython.sh

@@ -52,7 +52,7 @@ buildPython() {
 			pythonSdkFileName=python-$debianFlavor-$version.tar.gz
 	fi
 
-	if shouldBuildSdk python $pythonSdkFileName || shouldOverwriteSdk || shouldOverwritePythonSdk; then
+	if shouldBuildSdk python $pythonSdkFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk python; then
 		if ! $builtPythonPrereqs; then
 			buildPythonPrereqsImage
 		fi
@@ -84,14 +84,6 @@ buildPython() {
 	fi
 }
 
-shouldOverwritePythonSdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_PYTHON" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Building Python..."
 echo
 buildPlatform "$pythonPlatformDir/versionsToBuild.txt" buildPython

platforms/ruby/buildRuby.sh

@@ -42,7 +42,7 @@ buildRuby() {
 		rubySdkFileName=ruby-$debianFlavor-$version.tar.gz
 	fi 
 
-	if shouldBuildSdk ruby $rubySdkFileName || shouldOverwriteSdk || shouldOverwriteRubySdk; then
+	if shouldBuildSdk ruby $rubySdkFileName || shouldOverwriteSdk || shouldOverwritePlatformSdk ruby; then
 		if ! $builtRubyPrereqs; then
 			buildRubyPrereqsImage
 		fi
@@ -71,14 +71,6 @@ buildRuby() {
 	fi
 }
 
-shouldOverwriteRubySdk() {
-	if [ "$OVERWRITE_EXISTING_SDKS_RUBY" == "true" ]; then
-		return 0
-	else
-		return 1
-	fi
-}
-
 echo "Building Ruby..."
 echo
 buildPlatform "$rubyPlatformDir/versionsToBuild.txt" buildRuby

vsts/scripts/publishFilesToAzureStorage.sh

@@ -6,6 +6,8 @@
 
 set -ex
 
+declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && cd .. && pwd )
+source $REPO_DIR/platforms/__common.sh
 commit=$(git rev-parse HEAD)
 storageAccount="$1"
 
@@ -39,7 +41,21 @@ uploadFiles() {
             checksum=$(sha256sum $fileToUpload | cut -d " " -f 1)
         fi
         
-        az storage blob upload \
+        if shouldOverwriteSdk || shouldOverwritePlatformSdk $platform; then
+            az storage blob upload \
+            --name $fileName \
+            --file "$fileToUpload" \
+            --container-name $platform \
+            --account-name $storageAccount \
+            --metadata \
+                Buildnumber="$BUILD_BUILDNUMBER" \
+                Commit="$commit" \
+                Branch="$BUILD_SOURCEBRANCHNAME" \
+                Checksum="$checksum" \
+                $fileMetadata \
+            --overwrite true
+        else
+            az storage blob upload \
             --name $fileName \
             --file "$fileToUpload" \
             --container-name $platform \
@@ -50,6 +66,7 @@ uploadFiles() {
                 Branch="$BUILD_SOURCEBRANCHNAME" \
                 Checksum="$checksum" \
                 $fileMetadata
+        fi
     done
 }
 

vsts/scripts/publishSdksFromDevToProdStorageAccount.sh

@@ -9,6 +9,7 @@ set -ex
 declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && cd .. && pwd )
 source $REPO_DIR/build/__variables.sh
 source $REPO_DIR/build/__sdkStorageConstants.sh
+source $REPO_DIR/platforms/__common.sh
 
 azCopyDir="/tmp/azcopy-tool"
 
@@ -28,19 +29,17 @@ function blobExistsInProd() {
 	fi
 }
 
-function copyDefaultVersionFile() {
-    local defaultVersionFile="$1"
-    local platformName="$2"
-    "$azCopyDir/azcopy" copy \
-        "$defaultVersionFile" \
-        "$PROD_SDK_STORAGE_BASE_URL/$platformName/defaultVersion.txt$PROD_STORAGE_SAS_TOKEN"
-}
-
 function copyBlob() {
     local platformName="$1"
     local blobName="$2"
 
-    if blobExistsInProd $platformName $blobName; then
+    if shouldOverwriteSdk || shouldOverwritePlatformSdk $platformName; then
+        echo
+        echo "Blob '$blobName' exists in Prod storage container '$platformName'. Overwriting it..."
+        "$azCopyDir/azcopy" copy \
+            "$DEV_SDK_STORAGE_BASE_URL/$platformName/$blobName$DEV_STORAGE_SAS_TOKEN" \
+            "$PROD_SDK_STORAGE_BASE_URL/$platformName/$blobName$PROD_STORAGE_SAS_TOKEN" --overwrite true
+    elif blobExistsInProd $platformName $blobName; then
         echo
         echo "Blob '$blobName' already exists in Prod storage container '$platformName'. Skipping copying it..."
     else
@@ -55,14 +54,11 @@ function copyBlob() {
 function copyPlatformBlobsToProd() {
     local platformName="$1"
     local versionsFile="$REPO_DIR/platforms/$platformName/versionsToBuild.txt"
-    local defaultVersionFile="$REPO_DIR/platforms/$platformName/defaultVersion.txt"
 
     if [ "$platformName" == "php-composer" ]; then
         versionsFile="$REPO_DIR/platforms/php/composer/versionsToBuild.txt"
-        defaultVersionFile="$REPO_DIR/platforms/php/composer/defaultVersion.txt"
     elif [ "$platformName" == "maven" ]; then
         versionsFile="$REPO_DIR/platforms/java/maven/versionsToBuild.txt"
-        defaultVersionFile="$REPO_DIR/platforms/java/maven/defaultVersion.txt"
     fi
 
     # Here '3' is a file descriptor which is specifically used to read the versions file.
@@ -82,7 +78,7 @@ function copyPlatformBlobsToProd() {
         copyBlob "$platformName" "$platformName-buster-$version.tar.gz"
 	done 3< "$versionsFile"
 
-    copyDefaultVersionFile $defaultVersionFile "$platformName"
+    copyBlob "$platformName" defaultVersion.txt
 }
 
 if [ ! -f "$azCopyDir/azcopy" ]; then