Install sdks dynamically-part1 (#475)

.gitignore

@@ -267,4 +267,5 @@ src/startupscriptgenerator/**/vendor/
 
 # Go debugging related generated files
 src/startupscriptgenerator/src/**/__debug_bin
-src/startupscriptgenerator/src/**/run.sh
+src/startupscriptgenerator/src/**/run.sh
+!**/dynamic/Dockerfile

build/__sdkStorageConstants.sh

@@ -0,0 +1,6 @@
+# This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+USE_LATEST_VERSION='USE_LATEST_VERSION'
+SDK_STORAGE_BASE_URL_KEY_NAME='ORYX_SDK_STORAGE_BASE_URL'
+DEV_SDK_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
+PROD_SDK_STORAGE_BASE_URL='https://oryxsdks.blob.core.windows.net'

build/buildBuildImages.sh

@@ -14,6 +14,8 @@ source $REPO_DIR/build/__functions.sh
 source $REPO_DIR/build/__pythonVersions.sh # For PYTHON_BASE_TAG
 source $REPO_DIR/build/__phpVersions.sh    # For PHP_BUILD_BASE_TAG
 source $REPO_DIR/build/__nodeVersions.sh   # For YARN_CACHE_BASE_TAG
+source $REPO_DIR/build/__nodeVersions.sh   # For YARN_CACHE_BASE_TAG
+source $REPO_DIR/build/__sdkStorageConstants.sh
 
 declare -r BASE_TAG_BUILD_ARGS="--build-arg PYTHON_BASE_TAG=$PYTHON_BASE_TAG \
                                 --build-arg PHP_BUILD_BASE_TAG=$PHP_BUILD_BASE_TAG \
@@ -89,6 +91,8 @@ function buildDockerImage() {
 		--build-arg AGENTBUILD=$BUILD_SIGNED \
 		$BASE_TAG_BUILD_ARGS \
 		--build-arg AI_KEY=$APPLICATION_INSIGHTS_INSTRUMENTATION_KEY \
+		--build-arg SDK_STORAGE_ENV_NAME=$SDK_STORAGE_BASE_URL_KEY_NAME \
+		--build-arg SDK_STORAGE_BASE_URL_VALUE=$PROD_SDK_STORAGE_BASE_URL \
 		$ctxArgs \
 		-f "$dockerFileToBuild" \
 		.

build/buildRunTimeImages.sh

@@ -11,6 +11,7 @@ declare -r REPO_DIR=$( cd $( dirname "$0" ) && cd .. && pwd )
 # Load all variables
 source $REPO_DIR/build/__variables.sh
 source $REPO_DIR/build/__functions.sh
+source $REPO_DIR/build/__sdkStorageConstants.sh
 
 runtimeImagesSourceDir="$RUNTIME_IMAGES_SRC_DIR"
 runtimeSubDir="$1"
@@ -83,6 +84,8 @@ for dockerFile in $dockerFiles; do
         -f $dockerFile \
         -t $localImageTagName \
         --build-arg AI_KEY=$APPLICATION_INSIGHTS_INSTRUMENTATION_KEY \
+        --build-arg SDK_STORAGE_ENV_NAME=$SDK_STORAGE_BASE_URL_KEY_NAME \
+        --build-arg SDK_STORAGE_BASE_URL_VALUE=$PROD_SDK_STORAGE_BASE_URL \
         $args \
         $labels \
         .

build/constants.yaml

@@ -123,10 +123,27 @@
     - type: csharp
       directory: src/BuildScriptGenerator/Node
       namespace: Microsoft.Oryx.BuildScriptGenerator.Node
-- name: common-constants
+- name: sdk-storage-constants
   constants:
-    dev-storage-base-url: https://oryxsdksdev.blob.core.windows.net
+    use-latest-version: USE_LATEST_VERSION
+    sdk-storage-base-url-key-name: ORYX_SDK_STORAGE_BASE_URL
+    dev-sdk-storage-base-url: https://oryxsdksdev.blob.core.windows.net
+    prod-sdk-storage-base-url: https://oryxsdks.blob.core.windows.net
   outputs:
     - type: shell
-      directory: images
+      directory: build
       file-name-prefix: __
+    - type: csharp
+      directory: src/Common
+      namespace: Microsoft.Oryx.Common
+    - type: go
+      directory: src/startupscriptgenerator/src/common/consts
+- name: runtime-dynamic-installation-constants
+  constants:
+    node-installation-dir: '/tmp/oryx/nodejs'
+    dot-net-core-installation-dir: '/tmp/oryx/dotnet'
+    python-installation-root-dir: '/opt/python'
+    setup-script-location: '/tmp/oryx/setupEnv.sh'
+  outputs:
+    - type: go
+      directory: src/startupscriptgenerator/src/common/consts

images/__common.sh

@@ -10,7 +10,7 @@ function downloadFileAndVerifyChecksum() {
     echo "Downloading $platformName version '$version'..."
     curl \
         -D $headersFile \
-        -SL "$DEV_STORAGE_BASE_URL/$platformName/$platformName-$version.tar.gz" \
+        -SL "$DEV_SDK_STORAGE_BASE_URL/$platformName/$platformName-$version.tar.gz" \
         --output $downloadedFileName
 
     headerName="x-ms-meta-checksum"

images/__commonConstants.sh

@@ -1,3 +1,4 @@
 # This file was auto-generated from 'constants.yaml'. Changes may be overridden.
 
-DEV_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
+DEV_SDK_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
+PROD_SDK_STORAGE_BASE_URL='https://oryxsdks.blob.core.windows.net'

images/build/benv.sh

@@ -54,18 +54,63 @@ done < <(set | grep -i '^dotnet_version=')
 unset benvvar # Remove all traces of this part of the script
 
 # Oryx's paths come to the end of the PATH environment variable so that any user installed platform
-# sdk versions can be picked up. Here we are trying to find the first occurrence of a path like '/opt/'
-# (as in /opt/dotnet) and inserting a more specific provided path before it.
-# Example: (note that all Oryx related patlform paths come in the end)
-# /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/nodejs/6/bin:/opt/dotnet/sdks/2.2.401:/opt/oryx
+# sdk versions can be picked up. Here we are trying to find the first occurrence of a path like '/opt/oryx'
+# and inserting a more specific provided path after it.
+# Example: (note that all Oryx related patlform paths come after the typical debian paths)
+# /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/oryx:/opt/nodejs/6/bin:/opt/dotnet/sdks/2.2.401
 updatePath() {
-  local replacingText="$1:/opt/"
+  local replacingText=":/opt/oryx:$1:"
+  local lookUpText=":\/opt\/oryx:"
   local currentPath="$PATH"
-  local lookUpText="\/opt\/"
   local newPath=$(echo $currentPath | sed "0,/$lookUpText/ s##$replacingText#")
   export PATH="$newPath"
 }
 
+# NOTE: We handle .NET Core specially because there are 2 version types:
+# SDK version and Runtime version
+# For platforms other than dotnet, we look at a folder structure like '/opt/nodejs/10.14.1', but
+# for dotnet, it would be '/opt/dotnet/runtimes/10.14.1'
+# i.e Versioning of .NET Core is based on the runtime versions rather than sdk version
+benv-showSupportedVersionsErrorInfo() {
+  local userPlatformName="$1"
+  local platformDirName="$2"
+  local userSuppliedVersion="$3"
+  local builtInInstallDir="/opt/$platformDirName"
+  local dynamicInstallDir="/tmp/oryx/platforms/$platformDirName"
+
+  if [ "$platformDirName" == "dotnet" ]; then
+    builtInInstallDir="$builtInInstallDir/runtimes"
+    dynamicInstallDir="$dynamicInstallDir/runtimes"
+  fi
+
+  echo >&2 benv: "$userPlatformName" version \'$userSuppliedVersion\' not found\; choose one of:
+  benv-versions >&2 "$builtInInstallDir"
+
+  if [ -d "$dynamicInstallDir" ]; then
+    benv-versions >&2 "$dynamicInstallDir"
+  fi
+}
+
+benv-getPlatformDir() {
+  local platformDirName="$1"
+  local userSuppliedVersion="$2"
+  local builtInInstallDir="/opt/$platformDirName"
+  local dynamicInstallDir="/tmp/oryx/platforms/$platformDirName"
+  
+  if [ "$platformDirName" == "dotnet" ]; then
+    builtInInstallDir="$builtInInstallDir/runtimes"
+    dynamicInstallDir="$dynamicInstallDir/runtimes"
+  fi
+
+  if [ -d "$builtInInstallDir/$userSuppliedVersion" ]; then
+    echo "$builtInInstallDir/$userSuppliedVersion"
+  elif [ -d "$dynamicInstallDir/$userSuppliedVersion" ]; then
+    echo "$dynamicInstallDir/$userSuppliedVersion"
+  else
+    echo "NotFound"
+  fi
+}
+
 benv-versions() {
   local IFS=$' \r\n'
   local version
@@ -85,13 +130,13 @@ benv-resolve() {
 
   # Resolve node versions
   if matchesName "node" "$name" || matchesName "node_version" "$name" && [ "${value::1}" != "/" ]; then
-    if [ ! -d "/opt/nodejs/$value" ]; then
-      echo >&2 benv: node version \'$value\' not found\; choose one of:
-      benv-versions >&2 /opt/nodejs
+    platformDir=$(benv-getPlatformDir "nodejs" "$value")
+    if [ "$platformDir" == "NotFound" ]; then
+      benv-showSupportedVersionsErrorInfo "node" "nodejs" "$value"
       return 1
     fi
 
-    local DIR="/opt/nodejs/$value/bin"
+    local DIR="$platformDir/bin"
     updatePath "$DIR"
     export node="$DIR/node"
     export npm="$DIR/npm"
@@ -104,13 +149,13 @@ benv-resolve() {
 
   # Resolve npm versions
   if matchesName "npm" "$name" || matchesName "npm_version" "$name" && [ "${value::1}" != "/" ]; then
-    if [ ! -d "/opt/npm/$value" ]; then
-      echo >&2 benv: npm version \'$value\' not found\; choose one of:
-      benv-versions >&2 /opt/npm
+    platformDir=$(benv-getPlatformDir "npm" "$value")
+    if [ "$platformDir" == "NotFound" ]; then
+      benv-showSupportedVersionsErrorInfo "npm" "npm" "$value"
       return 1
     fi
 
-    local DIR="/opt/npm/$value"
+    local DIR="$platformDir"
     updatePath "$DIR"
     export npm="$DIR/npm"
     if [ -e "$DIR/npx" ]; then
@@ -122,13 +167,15 @@ benv-resolve() {
 
   # Resolve python versions
   if matchesName "python" "$name" || matchesName "python_version" "$name" && [ "${value::1}" != "/" ]; then
-    if [ ! -d "/opt/python/$value" ]; then
-      echo >&2 benv: python version \'$value\' not found\; choose one of:
-      benv-versions >&2 /opt/python
+    platformDir=$(benv-getPlatformDir "python" "$value")
+    if [ "$platformDir" == "NotFound" ]; then
+      benv-showSupportedVersionsErrorInfo "node" "python" "$value"
       return 1
     fi
 
-    local DIR="/opt/python/$value/bin"
+    export LD_LIBRARY_PATH="$platformDir/lib:$LD_LIBRARY_PATH"
+
+    local DIR="$platformDir/bin"
     updatePath "$DIR"
     if [ -e "$DIR/python2" ]; then
       export python="$DIR/python2"
@@ -145,13 +192,15 @@ benv-resolve() {
 
   # Resolve PHP versions
   if matchesName "php" "$name" || matchesName "php_version" "$name" && [ "${value::1}" != "/" ]; then
-    if [ ! -d "/opt/php/$value" ]; then
-      echo >&2 benv: php version \'$value\' not found\; choose one of:
-      benv-versions >&2 /opt/php
+    platformDir=$(benv-getPlatformDir "php" "$value")
+    if [ "$platformDir" == "NotFound" ]; then
+      benv-showSupportedVersionsErrorInfo "php" "php" "$value"
       return 1
     fi
 
-    local DIR="/opt/php/$value/bin"
+    export LD_LIBRARY_PATH="$platformDir/lib:$LD_LIBRARY_PATH"
+    
+    local DIR="$platformDir/bin"
     updatePath "$DIR"
     export php="$DIR/php"
 
@@ -160,14 +209,13 @@ benv-resolve() {
 
   # Resolve dotnet versions
   if matchesName "dotnet" "$name" || matchesName "dotnet_version" "$name" && [ "${value::1}" != "/" ]; then
-    local runtimesDir="/opt/dotnet/runtimes"
-    if [ ! -d "$runtimesDir/$value" ]; then
-      echo >&2 benv: dotnet version \'$value\' not found\; choose one of:
-      benv-versions >&2 $runtimesDir
+    runtimeDir=$(benv-getPlatformDir "dotnet" "$value")
+    if [ "$runtimeDir" == "NotFound" ]; then
+      benv-showSupportedVersionsErrorInfo "dotnet" "dotnet" "$value"
       return 1
     fi
 
-    local SDK_DIR=$(readlink $"$runtimesDir/$value/sdk")
+    local SDK_DIR=$(readlink $"$runtimeDir/sdk")
 
     toolsDir="$SDK_DIR/tools"
     if [ -d "$toolsDir" ]; then

images/build/slim.Dockerfile

@@ -2,6 +2,8 @@
 # These are deleted in the final stage of the build
 ARG IMAGES_DIR=/tmp/oryx/images
 ARG BUILD_DIR=/tmp/oryx/build
+ARG SDK_STORAGE_ENV_NAME
+ARG SDK_STORAGE_BASE_URL_VALUE
 # Determine where the image is getting built (DevOps agents or local)
 ARG AGENTBUILD
 
@@ -228,6 +230,8 @@ RUN chmod a+x /opt/buildscriptgen/GenerateBuildScript
 FROM python AS final
 ARG BUILD_DIR
 ARG IMAGES_DIR
+ARG SDK_STORAGE_ENV_NAME
+ARG SDK_STORAGE_BASE_URL_VALUE
 WORKDIR /
 
 ENV PATH="$PATH:/opt/oryx:/opt/nodejs/lts/bin:/opt/dotnet/sdks/lts:/opt/python/latest/bin:/opt/yarn/stable/bin:/opt/hugo"
@@ -267,4 +271,6 @@ LABEL com.microsoft.oryx.git-commit=${GIT_COMMIT}
 LABEL com.microsoft.oryx.build-number=${BUILD_NUMBER}
 LABEL com.microsoft.oryx.release-tag-name=${RELEASE_TAG_NAME}
 
+ENV ${SDK_STORAGE_ENV_NAME} ${SDK_STORAGE_BASE_URL_VALUE}
+
 ENTRYPOINT [ "benv" ]

images/runtime/dotnetcore/dynamic/Dockerfile

@@ -0,0 +1,44 @@
+# Startup script generator
+FROM golang:1.11-stretch as startupCmdGen
+# GOPATH is set to "/go" in the base image
+WORKDIR /go/src
+COPY src/startupscriptgenerator/src .
+ARG GIT_COMMIT=unspecified
+ARG BUILD_NUMBER=unspecified
+ARG RELEASE_TAG_NAME=unspecified
+ENV RELEASE_TAG_NAME=${RELEASE_TAG_NAME}
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+RUN ./build.sh dotnetcore /opt/startupcmdgen/startupcmdgen
+
+FROM debian:buster-slim
+
+ARG SDK_STORAGE_ENV_NAME
+ARG SDK_STORAGE_BASE_URL_VALUE
+ENV ${SDK_STORAGE_ENV_NAME} ${SDK_STORAGE_BASE_URL_VALUE}
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        \
+# .NET Core dependencies
+        libc6 \
+        libgcc1 \
+        libgssapi-krb5-2 \
+        libicu63 \
+        libssl1.1 \
+        libstdc++6 \
+        zlib1g \
+        lldb \
+        curl \
+        file \
+    && rm -rf /var/lib/apt/lists/*
+
+# Configure web servers to bind to port 80 when present
+ENV ASPNETCORE_URLS=http://+:80 \
+    # Enable detection of running in a container
+    DOTNET_RUNNING_IN_CONTAINER=true \
+    PATH="/tmp/oryx/dotnet:$PATH"
+
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen
+RUN ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx

images/runtime/node/dynamic/Dockerfile

@@ -0,0 +1,49 @@
+# Startup script generator
+FROM golang:1.11-stretch as startupCmdGen
+# GOPATH is set to "/go" in the base image
+WORKDIR /go/src
+COPY src/startupscriptgenerator/src .
+ARG GIT_COMMIT=unspecified
+ARG BUILD_NUMBER=unspecified
+ARG RELEASE_TAG_NAME=unspecified
+ENV RELEASE_TAG_NAME=${RELEASE_TAG_NAME}
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+RUN ./build.sh node /opt/startupcmdgen/startupcmdgen
+
+FROM oryx-run-base
+ARG IMAGES_DIR=/tmp/oryx/images
+RUN apt-get update \
+  && apt-get install -y \
+    unzip \
+  && rm -rf /var/lib/apt/lists/*
+  
+# NOTE: This is a list of keys for ALL Node versions and also Yarn package installs.
+# Receiving them once speeds up building of individual node versions as they all derive
+# from this image.
+
+# Gpg keys listed at https://github.com/nodejs/node
+RUN ${IMAGES_DIR}/receiveGpgKeys.sh \
+    6A010C5166006599AA17F08146C2130DFD2497F5
+
+ENV YARN_VERSION 1.17.3
+
+RUN curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \
+  && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \
+  && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \
+  && mkdir -p /opt \
+  && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \
+  && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \
+  && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \
+  && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz
+
+# Bake Application Insights key from pipeline variable into final image
+ARG AI_KEY
+ENV ORYX_AI_INSTRUMENTATION_KEY=${AI_KEY}
+ 
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen
+RUN ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx
+
+# Node wrapper is used to debug apps when node is executed indirectly, e.g. by npm.
+COPY src/startupscriptgenerator/src/node/wrapper/node /opt/node-wrapper/
+RUN chmod a+x /opt/node-wrapper/node

images/runtime/python/dynamic/Dockerfile

@@ -0,0 +1,27 @@
+# Startup script generator
+FROM golang:1.11-stretch as startupCmdGen
+# GOPATH is set to "/go" in the base image
+WORKDIR /go/src
+COPY src/startupscriptgenerator/src .
+ARG GIT_COMMIT=unspecified
+ARG BUILD_NUMBER=unspecified
+ARG RELEASE_TAG_NAME=unspecified
+ENV RELEASE_TAG_NAME=${RELEASE_TAG_NAME}
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+RUN ./build.sh python /opt/startupcmdgen/startupcmdgen
+
+FROM oryx-run-base
+ARG SDK_STORAGE_ENV_NAME
+ARG SDK_STORAGE_BASE_URL_VALUE
+ENV ${SDK_STORAGE_ENV_NAME} ${SDK_STORAGE_BASE_URL_VALUE}
+
+ARG IMAGES_DIR=/tmp/oryx/images
+
+# Bake Application Insights key from pipeline variable into final image
+ARG AI_KEY
+ENV ORYX_AI_INSTRUMENTATION_KEY=${AI_KEY}
+RUN ${IMAGES_DIR}/runtime/python/install-dependencies.sh
+RUN rm -rf /tmp/oryx
+COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen
+RUN  ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx

images/runtime/python/install-dependencies.sh

@@ -44,11 +44,5 @@ Threading=1
 UsageCount=1
 EOL
 
-pip install --upgrade pip
-pip install gunicorn # The default WSGI server
-pip install ptvsd
-
-ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx
-
 # Clean up for apt. Keeping at the very end to make sure it runs after every apt-get install.
 rm -rf /var/lib/apt/lists/*

images/runtime/python/template.Dockerfile

@@ -33,5 +33,10 @@ ENV PATH="/opt/python/%PYTHON_MAJOR_VERSION%/bin:${PATH}"
 ARG AI_KEY
 ENV ORYX_AI_INSTRUMENTATION_KEY=${AI_KEY}
 RUN ${IMAGES_DIR}/runtime/python/install-dependencies.sh
+RUN pip install --upgrade pip \
+    && pip install gunicorn \
+    && pip install ptvsd \
+    && ln -s /opt/startupcmdgen/startupcmdgen /usr/local/bin/oryx
+
 RUN rm -rf /tmp/oryx
 COPY --from=startupCmdGen /opt/startupcmdgen/startupcmdgen /opt/startupcmdgen/startupcmdgen