Add Ruby platform pipeline (#746)

build/__rubyVersions.sh

@@ -0,0 +1,7 @@
+# This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+GEM_VERSION='3.1.4'
+RUBY27_VERSION='2.7.1'
+RUBY27_TAR_SHA256='b224f9844646cc92765df8288a46838511c1cec5b550d8874bd4686a904fcee7'
+RUBY26_VERSION='2.6.6'
+RUBY26_TAR_SHA256='5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f'

build/buildPlatformBinaries.sh

@@ -23,6 +23,9 @@ case $PLATFORM_TO_BUILD in
 		;;
 	'nodejs')
 		"$platformsDir/nodejs/getNode.sh"
-		;;            
+		;;
+	'ruby')
+		"$platformsDir/ruby/buildRuby.sh"
+		;;
 	*) echo "Unknown image directory";;
 esac

build/constants.yaml

@@ -107,6 +107,21 @@
     - type: shell
       directory: build
       file-name-prefix: __
+- name: ruby-versions
+  constants:
+    gem-version: 3.1.4
+    # hashes are for .tar.xz
+    ruby27-version: 2.7.1
+    ruby27-tar-sha256: b224f9844646cc92765df8288a46838511c1cec5b550d8874bd4686a904fcee7
+    ruby26-version: 2.6.6
+    ruby26-tar-sha256: 5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f
+  outputs:
+    - type: csharp
+      directory: src/BuildScriptGenerator
+      namespace: Microsoft.Oryx.BuildScriptGenerator.Common
+    - type: shell
+      directory: build
+      file-name-prefix: __
 - name: file-paths
   constants:
     compressed-output-file-name: oryx_output.tar.gz

platforms/ruby/Dockerfile

@@ -0,0 +1,9 @@
+FROM ruby-build-prereqs
+WORKDIR /usr/src/ruby
+ARG RUBY_VERSION
+ARG GEM_VERSION
+ARG RUBY_SHA256
+RUN RUBY_VERSION=$RUBY_VERSION \
+    GEM_VERSION=$GEM_VERSION \
+    RUBY_SHA256=$RUBY_SHA256 \
+    /tmp/build.sh

platforms/ruby/buildRuby.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT license.
+# --------------------------------------------------------------------------------------------
+
+set -ex
+
+declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && cd .. && pwd )
+
+source $REPO_DIR/platforms/__common.sh
+source $REPO_DIR/build/__rubyVersions.sh
+
+rubyPlatformDir="$REPO_DIR/platforms/ruby"
+targetDir="$volumeHostDir/ruby"
+mkdir -p "$targetDir"
+
+builtRubyPrereqs=false
+buildRubyPrereqsImage() {
+	if ! $builtRubyPrereqs; then
+		echo "Building Ruby pre-requisites image..."
+		echo
+		docker build -f "$rubyPlatformDir/prereqs/Dockerfile" -t "ruby-build-prereqs" $REPO_DIR
+		builtRubyPrereqs=true
+	fi
+}
+
+buildRuby() {
+	local version="$1"
+	local sha="$2"
+	local imageName="oryx/ruby"
+
+	if shouldBuildSdk ruby ruby-$version.tar.gz || shouldOverwriteSdk || shouldOverwriteRubySdk; then
+		if ! $builtRubyPrereqs; then
+			buildRubyPrereqsImage
+		fi
+        
+        echo "Building Ruby version '$version' in a docker image..."
+		echo
+
+		if [ -z "$dockerFile" ]; then
+			# Use common docker file
+			dockerFile="$rubyPlatformDir/Dockerfile"
+		else
+			dockerFile="$rubyPlatformDir/$dockerFile"
+		fi
+
+		docker build \
+			-f "$rubyPlatformDir/Dockerfile" \
+			--build-arg RUBY_VERSION=$version \
+            --build-arg RUBY_SHA256=$sha \
+			--build-arg GEM_VERSION=$GEM_VERSION \
+			-t $imageName \
+			$REPO_DIR
+
+		getSdkFromImage $imageName "$targetDir"
+
+		echo "Version=$version" >> "$targetDir/ruby-$version-metadata.txt"
+	fi
+}
+
+shouldOverwriteRubySdk() {
+	if [ "$OVERWRITE_EXISTING_SDKS_RUBY" == "true" ]; then
+		return 0
+	else
+		return 1
+	fi
+}
+
+echo "Building Ruby..."
+echo
+buildPlatform "$rubyPlatformDir/versionsToBuild.txt" buildRuby
+
+# Write the default version
+cp "$rubyPlatformDir/defaultVersion.txt" $targetDir 

platforms/ruby/defaultVersion.txt

@@ -0,0 +1 @@
+2.7

platforms/ruby/prereqs/Dockerfile

@@ -0,0 +1,5 @@
+# Install Ruby build prerequisites
+FROM buildpack-deps:stretch AS ruby-build-prereqs
+COPY build/__rubyVersions.sh /tmp/
+COPY platforms/ruby/prereqs/build.sh /tmp/
+RUN chmod +x /tmp/build.sh

platforms/ruby/prereqs/build.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# This script is referenced from official docker library: 
+# https://github.com/docker-library/ruby/blob/master/Dockerfile-debian.template
+# some of ruby's build scripts are written in ruby
+# we purge system ruby later to make sure our final image uses what we just built
+
+set -eux
+
+LANG=C.UTF-8
+RUBY_MAJOR_VERSION=${RUBY_VERSION:0:3}
+INSTALLATION_PREFIX=/opt/ruby/$RUBY_VERSION
+
+# skip installing gem documentation
+set -eux; \
+	mkdir -p $INSTALLATION_PREFIX/etc; \
+	{ \
+		echo 'install: --no-document'; \
+		echo 'update: --no-document'; \
+	} >> $INSTALLATION_PREFIX/etc/gemrc
+
+set -eux; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		bison \
+		dpkg-dev \
+		libgdbm-dev \
+		ruby \
+	; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+	wget -O ruby.tar.xz "https://cache.ruby-lang.org/pub/ruby/$RUBY_MAJOR_VERSION/ruby-$RUBY_VERSION.tar.xz"; \
+	echo "$RUBY_SHA256 *ruby.tar.xz" | sha256sum --check --strict; \
+	\
+	mkdir -p /usr/src/ruby; \
+	tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1; \
+	rm ruby.tar.xz; \
+	\
+	cd /usr/src/ruby; \
+	\
+
+# hack in "ENABLE_PATH_CHECK" disabling to suppress:
+#   warning: Insecure world writable dir
+	{ \
+		echo '#define ENABLE_PATH_CHECK 0'; \
+		echo; \
+		cat file.c; \
+	} > file.c.new; \
+	mv file.c.new file.c; \
+	\
+	autoconf; \
+	gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \
+	./configure \
+        --prefix=$INSTALLATION_PREFIX \
+		--build="$gnuArch" \
+		--disable-install-doc \
+		--enable-shared \
+	; \
+	make -j "$(nproc)"; \
+	make install; \
+	\
+	apt-mark auto '.*' > /dev/null; \
+	apt-mark manual $savedAptMark > /dev/null; \
+	find $INSTALLATION_PREFIX -type f -executable -not \( -name '*tkinter*' \) -exec ldd '{}' ';' \
+		| awk '/=>/ { print $(NF-1) }' \
+		| sort -u \
+		| xargs -r dpkg-query --search \
+		| cut -d: -f1 \
+		| sort -u \
+		| xargs -r apt-mark manual \
+	; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	\
+	cd /; \
+	rm -r /usr/src/ruby; \
+
+rubyBinDir="$INSTALLATION_PREFIX/bin"
+
+echo
+echo "Contents of '$rubyBinDir':"
+ls -l $rubyBinDir
+echo
+
+# make sure bundled "rubygems" is older than GEM_VERSION (https://github.com/docker-library/ruby/issues/246)
+$rubyBinDir/ruby -e 'exit(Gem::Version.create(ENV["GEM_VERSION"]) > Gem::Version.create(Gem::VERSION))'; \
+$rubyBinDir/gem update --system "$GEM_VERSION" && rm -r /root/.gem/; \
+
+# verify we have no "ruby" packages installed
+! dpkg -l | grep -i $rubyBinDir/ruby; \
+[ "$(command -v $rubyBinDir/ruby)" = "$rubyBinDir/ruby" ]; \
+
+# rough smoke test
+$rubyBinDir/ruby --version; \
+$rubyBinDir/gem --version; \
+$rubyBinDir/bundle --version
+
+# don't create ".bundle" in all our apps
+GEM_HOME=$INSTALLATION_PREFIX/bundle
+BUNDLE_SILENCE_ROOT_WARNING=1
+BUNDLE_APP_CONFIG="$GEM_HOME"
+PATH=$GEM_HOME/bin:$PATH
+
+# adjust permissions of a few directories for running "gem install" as an arbitrary user
+mkdir -p "$GEM_HOME"; \
+chmod 777 "$GEM_HOME" 
+
+compressedSdkDir="/tmp/compressedSdk"
+mkdir -p $compressedSdkDir
+cd "$INSTALLATION_PREFIX"
+tar -zcf $compressedSdkDir/ruby-$RUBY_VERSION.tar.gz .

platforms/ruby/versionsToBuild.txt

@@ -0,0 +1,3 @@
+# version, sha
+2.7.1, b224f9844646cc92765df8288a46838511c1cec5b550d8874bd4686a904fcee7,
+2.6.6, 5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f

src/BuildScriptGenerator/RubyVersions.cs

@@ -0,0 +1,13 @@
+// This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Common
+{
+    public static class RubyVersions
+    {
+        public const string GemVersion = "3.1.4";
+        public const string Ruby27Version = "2.7.1";
+        public const string Ruby27TarSha256 = "b224f9844646cc92765df8288a46838511c1cec5b550d8874bd4686a904fcee7";
+        public const string Ruby26Version = "2.6.6";
+        public const string Ruby26TarSha256 = "5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f";
+    }
+} 

tests/Oryx.Integration.Tests/StorageAccountSanityTestBase.cs

@@ -114,7 +114,7 @@ namespace Oryx.Integration.Tests
             // Assert
             Assert.Equal(expectedVersion, actualVersion);
         }
-
+        
         [Fact]
         public void PhpComposerCoreContainer_HasExpectedListOfBlobs()
         {
@@ -142,6 +142,33 @@ namespace Oryx.Integration.Tests
             Assert.Equal(expectedVersion, actualVersion);
         }
 
+        [Fact]
+        public void RubyContainer_HasExpectedListOfBlobs()
+        {
+            // Arrange & Act
+            var platformName = "ruby";
+            var actualVersions = GetVersionsFromContainer(platformName, "version");
+            var expectedVersions = GetListOfVersionsToBuild(platformName);
+
+            // Assert
+            foreach (var expectedVersion in expectedVersions)
+            {
+                Assert.Contains(expectedVersion, actualVersions);
+            }
+        }
+
+        [Fact]
+        public void RubyContainer_HasExpectedDefaultVersion()
+        {
+            // Arrange & Act
+            var platformName = "ruby";
+            var actualVersion = GetDefaultVersionFromContainer(platformName);
+            var expectedVersion = GetDefaultVersion(platformName);
+
+            // Assert
+            Assert.Equal(expectedVersion, actualVersion);
+        }
+
         private XDocument GetMetadata(string platformName)
         {
             var url = string.Format(SdkStorageConstants.ContainerMetadataUrlFormat, _storageUrl, platformName);

vsts/pipelines/PlatformBinaries/ruby.yml

@@ -0,0 +1,37 @@
+variables:
+  - group: Oryx
+  
+stages:
+- stage: Build
+  jobs:
+  - job: Ruby
+    timeoutInMinutes: 250
+    pool:
+      name: OryxLinux
+    steps:
+    - template: ../templates/_platformBinariesTemplate.yml
+      parameters:
+        platformName: 'ruby'
+  
+- stage: Release
+  dependsOn: Build
+  jobs:
+  - job: Publish_Platform_Binaries
+    timeoutInMinutes: 250
+    displayName: Publish to Azure Blob Storage
+    pool:
+      name: OryxLinux
+    steps:
+    - template: ../templates/_platformBinariesReleaseTemplate.yml
+  
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - /*
+    include:
+    - platforms/ruby
+    - vsts/PlatformBinaries/ruby.yml

vsts/pipelines/baseImages/ruby.yml

@@ -0,0 +1,36 @@
+variables:
+    ascName: OryxMCR
+    acrName: oryxdevmcr.azurecr.io
+    skipComponentGovernanceDetection: true
+  
+jobs:
+- template: ../templates/_buildimageBasesJobTemplate.yml
+  parameters:
+    displayName: Build ruby runtime stretch base images
+    scriptPath: ./build/buildRunTimeImageBases.sh
+    imageDir: ruby
+    imageDebianFlavor: stretch
+    artifactsFileName: ruby-runtimeimage-bases-stretch.txt
+    jobName: Build_Ruby_Stretch_BaseImages
+
+- template: ../templates/_buildimageBasesJobTemplate.yml
+  parameters:
+    displayName: Build ruby runtime buster base images
+    scriptPath: ./build/buildRunTimeImageBases.sh
+    imageDir: ruby
+    imageDebianFlavor: buster
+    artifactsFileName: ruby-runtimeimage-bases-buster.txt
+    jobName: Build_Ruby_Buster_BaseImages
+
+- job: Release_RubypRuntimeBaseImage
+  dependsOn: 
+    - Build__Ruby_Stretch_BaseImages
+    - Build_Ruby_Buster_BaseImages
+  displayName: Push images to MCR
+  timeoutInMinutes: 250
+  pool:
+    name: OryxLinux
+  steps:
+    - template: ../templates/_releaseBaseImagesStepTemplate.yml
+      parameters:
+        baseImageName: 'ruby'

vsts/pipelines/templates/_integrationJobTemplate.yml

@@ -99,6 +99,31 @@ jobs:
   - template: _setReleaseTag.yml
   - template: _buildTemplate.yml
 
+- job: Job_RubyIntegrationTests
+  displayName: Run Ruby Integration Tests
+  dependsOn:
+    - Job_BuildImage
+    - Job_RuntimeImages
+  pool:
+    name: OryxLinux
+  variables:
+    skipComponentGovernanceDetection: true
+  timeoutInMinutes: 300
+  steps:
+  - script: |
+      echo "##vso[task.setvariable variable=BuildBuildImages;]false"
+      echo "##vso[task.setvariable variable=BuildRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestBuildImages;]false"
+      echo "##vso[task.setvariable variable=TestRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=ruby"
+      echo "##vso[task.setvariable variable=TestIntegration;]true"
+      echo "##vso[task.setvariable variable=PushBuildImages;]false"
+      echo "##vso[task.setvariable variable=PushRuntimeImages;]false"
+      echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false"
+    displayName: 'Set variables'
+  - template: _setReleaseTag.yml
+  - template: _buildTemplate.yml
+
 - job: Job_DbIntegrationTests
   displayName: Run Database Integration Tests
   dependsOn:

vsts/pipelines/templates/_releaseBaseImagesJobTemplate.yml

@@ -39,6 +39,16 @@ jobs:
     parameters:
       baseImageName: 'node'
 
+- job: Release_RubyBaseImage
+  displayName: Push Ruby Base Image to MCR
+  timeoutInMinutes: 300
+  pool:
+    name: OryxLinux
+  steps:
+  - template: _releaseBaseImagesStepTemplate.yml
+    parameters:
+      aseImageName: 'ruby'
+
 - job: Release_YarnCacheBuildBaseImage
   displayName: Push Yarn Cache Build Base Image to MCR
   timeoutInMinutes: 250

vsts/scripts/publishFilesToAzureStorage.sh

@@ -49,7 +49,7 @@ uploadFiles() {
     done
 }
 
-platforms=("nodejs" "python" "dotnet" "php" "php-composer")
+platforms=("nodejs" "python" "dotnet" "php" "php-composer" "ruby")
 for platform in "${platforms[@]}"
 do
     uploadFiles $platform

vsts/scripts/publishSdksFromDevToProdStorageAccount.sh

@@ -96,3 +96,4 @@ copyPlatformBlobsToProd "python"
 copyPlatformBlobsToProd "nodejs"
 copyPlatformBlobsToProd "php"
 copyPlatformBlobsToProd "php-composer"
+copyPlatformBlobsToProd "ruby"

vsts/scripts/tagBaseImagesForRelease.sh

@@ -96,6 +96,12 @@ then
   echo $imageName
   retagImageWithStagingRepository dotnetcore-runtimeimage-bases-buster.txt $imageName buster
   retagImageWithStagingRepository dotnetcore-runtimeimage-bases-stretch.txt $imageName stretch
+elif [ "$imageName" == "ruby" ]
+then
+  echo ""
+  echo $imageName
+  retagImageWithStagingRepository ruby-runtimeimage-bases-buster.txt $imageName buster
+  retagImageWithStagingRepository ruby-runtimeimage-bases-stretch.txt $imageName stretch
 else
   echo "ImageName $imageName is invalid/not supported.. "
   exit 1