Install sdks dynamically-part2. Python sdks installation (#478)

README.md

@@ -60,7 +60,7 @@ placed; if not specified the source directory is used for output as well.
 
 For all options, specify `oryx --help`.
 
-### `oryx -appPath`
+### `oryx create-script -appPath`
 
 When `oryx` is run in the runtime images it generates a start script named
 run.sh, by default in the same folder as the compiled artifact.
@@ -92,7 +92,7 @@ docker run --detach --rm \
     --env PORT=8080 \
     --publish 8080:8080 \
     'docker.io/oryxprod/node-10.14:latest' \
-    sh -c 'oryx -appPath /app && /run.sh'
+    sh -c 'oryx create-script -appPath /app && /run.sh'
 ```
 
 # Components

build/__sdkStorageConstants.sh

@@ -1,6 +1,5 @@
 # This file was auto-generated from 'constants.yaml'. Changes may be overridden.
 
-USE_LATEST_VERSION='USE_LATEST_VERSION'
 SDK_STORAGE_BASE_URL_KEY_NAME='ORYX_SDK_STORAGE_BASE_URL'
 DEV_SDK_STORAGE_BASE_URL='https://oryxsdksdev.blob.core.windows.net'
 PROD_SDK_STORAGE_BASE_URL='https://oryxsdks.blob.core.windows.net'

build/constants.yaml

@@ -125,7 +125,6 @@
       namespace: Microsoft.Oryx.BuildScriptGenerator.Node
 - name: sdk-storage-constants
   constants:
-    use-latest-version: USE_LATEST_VERSION
     sdk-storage-base-url-key-name: ORYX_SDK_STORAGE_BASE_URL
     dev-sdk-storage-base-url: https://oryxsdksdev.blob.core.windows.net
     prod-sdk-storage-base-url: https://oryxsdks.blob.core.windows.net

src/BuildScriptGenerator/BaseBashBuildScript.sh.tpl

@@ -63,6 +63,12 @@ echo "Source directory     : $SOURCE_DIR"
 echo "Destination directory: $DESTINATION_DIR"
 echo
 
+{{ for Snippet in PlatformInstallationScriptSnippets }}
+{{~ Snippet }}
+{{ end }}
+
+cd "$SOURCE_DIR"
+
 {{ if BenvArgs | IsNotBlank }}
 if [ -f {{ BenvPath }} ]; then
 	source {{ BenvPath }} {{ BenvArgs }}

src/BuildScriptGenerator/BaseBashBuildScriptProperties.cs

@@ -66,5 +66,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         /// Gets or set the path to benv file.
         /// </summary>
         public string BenvPath { get; set; }
+
+        /// <summary>
+        /// Gets or sets the list of bash script snippets which install the platform binaries.
+        /// </summary>
+        public IEnumerable<string> PlatformInstallationScriptSnippets { get; set; }
     }
 }

src/BuildScriptGenerator/BuildScriptGenerator.csproj

@@ -21,9 +21,11 @@
   <Import Project="$(MSBuildThisFileDirectory)\..\CommonFiles\AssemblyVersion.proj" />
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="2.2.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="2.2.0" />
     <PackageReference Include="Nett" Version="0.13.0" />
     <PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
     <PackageReference Include="Scriban.Signed" Version="1.2.9" />

src/BuildScriptGenerator/BuildScriptGeneratorServiceCollectionExtensions.cs

@@ -4,8 +4,13 @@
 // --------------------------------------------------------------------------------------------
 
 using System;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
+using Polly;
+using Polly.Extensions.Http;
 
 namespace Microsoft.Oryx.BuildScriptGenerator
 {
@@ -28,6 +33,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator
             services.AddSingleton<IScriptExecutor, DefaultScriptExecutor>();
             services.AddSingleton<IEnvironmentSettingsProvider, DefaultEnvironmentSettingsProvider>();
             services.AddSingleton<IRunScriptGenerator, DefaultRunScriptGenerator>();
+            services.AddHttpClient("general", httpClient =>
+            {
+                // NOTE: Setting user agent is required to avoid receiving 403 Forbidden response.
+                httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("oryx", "1.0"));
+            }).AddPolicyHandler(GetRetryPolicy());
 
             // Add all checkers (platform-dependent + platform-independent)
             foreach (Type type in typeof(BuildScriptGeneratorServiceCollectionExtensions).Assembly.GetTypes())
@@ -40,5 +50,15 @@ namespace Microsoft.Oryx.BuildScriptGenerator
 
             return services;
         }
+
+        private static IAsyncPolicy<HttpResponseMessage> GetRetryPolicy()
+        {
+            return HttpPolicyExtensions
+                .HandleTransientHttpError()
+                .OrResult(msg => msg.StatusCode == HttpStatusCode.NotFound)
+                .WaitAndRetryAsync(
+                    retryCount: 6,
+                    retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
+        }
     }
 }

src/BuildScriptGenerator/BuildScriptSnippet.cs

@@ -31,5 +31,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         /// Gets or sets a value indicating whether this snippet represents a full script.
         /// </summary>
         public bool IsFullScript { get; set; }
+
+        public string PlatformInstallationScriptSnippet { get; set; }
     }
 }

src/BuildScriptGenerator/Constants.cs

@@ -24,5 +24,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
 
         public const string True = "true";
         public const string False = "false";
+
+        public const string TemporaryInstallationDirectoryRoot = "/tmp/oryx/platforms";
     }
 }

src/BuildScriptGenerator/DefaultBuildScriptGenerator.cs

@@ -261,6 +261,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
                 ManifestDir = context.ManifestDir,
                 BuildProperties = buildProperties,
                 BenvPath = FilePaths.Benv,
+                PlatformInstallationScriptSnippets = snippets.Select(s => s.PlatformInstallationScriptSnippet),
             };
 
             LogScriptIfGiven("pre-build", buildScriptProps.PreBuildCommand);

src/BuildScriptGenerator/ManifestFilePropertyKeys.cs

@@ -10,5 +10,15 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         public const string ZipAllOutput = nameof(ZipAllOutput);
 
         public const string OperationId = nameof(OperationId);
+
+        public const string PhpVersion = nameof(PhpVersion);
+
+        public const string NodeVersion = nameof(NodeVersion);
+
+        public const string DotNetCoreRuntimeVersion = nameof(DotNetCoreRuntimeVersion);
+
+        public const string DotNetCoreSdkVersion = nameof(DotNetCoreSdkVersion);
+
+        internal const string PythonVersion = nameof(PythonVersion);
     }
 }

src/BuildScriptGenerator/Options/BuildScriptGeneratorOptions.cs

@@ -28,5 +28,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator
         public IDictionary<string, string> Properties { get; set; }
 
         public string ManifestDir { get; set; }
+
+        public bool EnableDynamicInstall { get; set; }
     }
 }

src/BuildScriptGenerator/PlatformInstallerBase.cs

@@ -0,0 +1,100 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System;
+using System.Linq;
+using System.Text;
+using Microsoft.Extensions.Options;
+using Microsoft.Oryx.Common;
+
+namespace Microsoft.Oryx.BuildScriptGenerator
+{
+    public abstract class PlatformInstallerBase
+    {
+        protected readonly BuildScriptGeneratorOptions _commonOptions;
+        protected readonly IEnvironment _environment;
+
+        public PlatformInstallerBase(IOptions<BuildScriptGeneratorOptions> commonOptions, IEnvironment environment)
+        {
+            _commonOptions = commonOptions.Value;
+            _environment = environment;
+        }
+
+        public abstract string GetInstallerScriptSnippet(string version);
+
+        public abstract bool IsVersionAlreadyInstalled(string version);
+
+        protected string GetInstallerScriptSnippet(
+            string platformName,
+            string version,
+            string directoryToInstall = null)
+        {
+            var sdkStorageBaseUrl = GetPlatformBinariesStorageBaseUrl();
+
+            var versionDirInTemp = directoryToInstall;
+            if (string.IsNullOrEmpty(versionDirInTemp))
+            {
+                versionDirInTemp = $"{Constants.TemporaryInstallationDirectoryRoot}/{platformName}/{version}";
+            }
+
+            var tarFile = $"{version}.tar.gz";
+            var snippet = new StringBuilder();
+            snippet
+                .AppendLine()
+                .AppendLine("PLATFORM_SETUP_START=$SECONDS")
+                .AppendLine("echo")
+                .AppendLine($"echo Downloading and installing {platformName} version '{version}'...")
+                .AppendLine($"rm -rf {versionDirInTemp}")
+                .AppendLine($"mkdir -p {versionDirInTemp}")
+                .AppendLine($"cd {versionDirInTemp}")
+                .AppendLine("PLATFORM_BINARY_DOWNLOAD_START=$SECONDS")
+                .AppendLine(
+                $"curl -D headers.txt -SL \"{sdkStorageBaseUrl}/{platformName}/{platformName}-{version}.tar.gz\" " +
+                $"--output {tarFile} >/dev/null 2>&1")
+                .AppendLine("PLATFORM_BINARY_DOWNLOAD_ELAPSED_TIME=$(($SECONDS - $PLATFORM_BINARY_DOWNLOAD_START))")
+                .AppendLine("echo \"Downloaded in $PLATFORM_BINARY_DOWNLOAD_ELAPSED_TIME sec(s).\"")
+                .AppendLine("headerName=\"x-ms-meta-checksum\"")
+                .AppendLine("checksumHeader=$(cat headers.txt | grep $headerName: | tr -d '\\r')")
+                .AppendLine("rm -f headers.txt")
+                .AppendLine("checksumValue=${checksumHeader#\"$headerName: \"}")
+                .AppendLine($"echo \"$checksumValue {version}.tar.gz\" | sha512sum -c - >/dev/null 2>&1")
+                .AppendLine($"tar -xzf {tarFile} -C .")
+                .AppendLine($"rm -f {tarFile}")
+                .AppendLine("PLATFORM_SETUP_ELAPSED_TIME=$(($SECONDS - $PLATFORM_SETUP_START))")
+                .AppendLine("echo \"Done in $PLATFORM_SETUP_ELAPSED_TIME sec(s).\"")
+                .AppendLine("echo");
+            return snippet.ToString();
+        }
+
+        protected bool IsVersionInstalled(string lookupVersion, string[] installationDirs)
+        {
+            foreach (var installationDir in installationDirs)
+            {
+                var versionsFromDisk = VersionProviderHelper.GetVersionsFromDirectory(installationDir);
+                if (versionsFromDisk.Any(onDiskVersion
+                    => string.Equals(lookupVersion, onDiskVersion, StringComparison.OrdinalIgnoreCase)))
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        private string GetPlatformBinariesStorageBaseUrl()
+        {
+            var platformBinariesStorageBaseUrl = _environment.GetEnvironmentVariable(
+                SdkStorageConstants.SdkStorageBaseUrlKeyName);
+            if (string.IsNullOrEmpty(platformBinariesStorageBaseUrl))
+            {
+                throw new InvalidOperationException(
+                    $"Environment variable '{SdkStorageConstants.SdkStorageBaseUrlKeyName}' is required.");
+            }
+
+            platformBinariesStorageBaseUrl = platformBinariesStorageBaseUrl.TrimEnd('/');
+            return platformBinariesStorageBaseUrl;
+        }
+    }
+}

src/BuildScriptGenerator/PlatformVersionInfo.cs

@@ -0,0 +1,39 @@
+using System.Collections.Generic;
+
+namespace Microsoft.Oryx.BuildScriptGenerator
+{
+    public class PlatformVersionInfo
+    {
+        public IEnumerable<string> SupportedVersions { get; private set; }
+
+        public string DefaultVersion { get; private set; }
+
+        public PlatformVersionSourceType PlatformVersionSourceType { get; private set; }
+
+        private PlatformVersionInfo() { }
+
+        public static PlatformVersionInfo CreateOnDiskVersionInfo(
+            IEnumerable<string> supportedVersions,
+            string defaultVersion)
+        {
+            return new PlatformVersionInfo
+            {
+                SupportedVersions = supportedVersions,
+                DefaultVersion = defaultVersion,
+                PlatformVersionSourceType = PlatformVersionSourceType.OnDisk,
+            };
+        }
+
+        public static PlatformVersionInfo CreateAvailableOnWebVersionInfo(
+            IEnumerable<string> supportedVersions,
+            string defaultVersion)
+        {
+            return new PlatformVersionInfo
+            {
+                SupportedVersions = supportedVersions,
+                DefaultVersion = defaultVersion,
+                PlatformVersionSourceType = PlatformVersionSourceType.AvailableOnWeb,
+            };
+        }
+    }
+}

src/BuildScriptGenerator/PlatformVersionSourceType.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.Oryx.BuildScriptGenerator
+{
+    public enum PlatformVersionSourceType
+    {
+        OnDisk,
+        AvailableOnWeb
+    }
+}

src/BuildScriptGenerator/Python/PythonLanguageDetector.cs

@@ -7,7 +7,6 @@ using System;
 using System.IO;
 using System.Linq;
 using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Options;
 using Microsoft.Oryx.BuildScriptGenerator.Exceptions;
 using Microsoft.Oryx.Common.Extensions;
 
@@ -15,21 +14,16 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
 {
     internal class PythonLanguageDetector : ILanguageDetector
     {
-        private readonly PythonScriptGeneratorOptions _pythonScriptGeneratorOptions;
         private readonly IPythonVersionProvider _versionProvider;
         private readonly ILogger<PythonLanguageDetector> _logger;
-        private readonly IStandardOutputWriter _writer;
 
         public PythonLanguageDetector(
-            IOptions<PythonScriptGeneratorOptions> options,
             IPythonVersionProvider pythonVersionProvider,
             ILogger<PythonLanguageDetector> logger,
             IStandardOutputWriter writer)
         {
-            _pythonScriptGeneratorOptions = options.Value;
             _versionProvider = pythonVersionProvider;
             _logger = logger;
-            _writer = writer;
         }
 
         public LanguageDetectorResult Detect(RepositoryContext context)
@@ -68,22 +62,29 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
 
         private string VerifyAndResolveVersion(string version)
         {
+            // Get the versions either from disk or on the web
+            var versionInfo = _versionProvider.GetVersionInfo();
+
+            // Get the default version. This could be having just the major or major.minor version.
+            // So try getting the latest version of the default version.
             if (string.IsNullOrEmpty(version))
             {
-                return _pythonScriptGeneratorOptions.PythonDefaultVersion;
+                version = versionInfo.DefaultVersion;
             }
 
             var maxSatisfyingVersion = SemanticVersionResolver.GetMaxSatisfyingVersion(
                 version,
-                _versionProvider.SupportedPythonVersions);
+                versionInfo.SupportedVersions);
 
             if (string.IsNullOrEmpty(maxSatisfyingVersion))
             {
                 var exc = new UnsupportedVersionException(
                     PythonConstants.PythonName,
                     version,
-                    _versionProvider.SupportedPythonVersions);
-                _logger.LogError(exc, $"Exception caught, the version '{version}' is not supported for the Python platform.");
+                    versionInfo.SupportedVersions);
+                _logger.LogError(
+                    exc,
+                    $"Exception caught, the version '{version}' is not supported for the Python platform.");
                 throw exc;
             }
 

src/BuildScriptGenerator/Python/PythonManifestFilePropertyKeys.cs

@@ -5,7 +5,7 @@
 
 namespace Microsoft.Oryx.BuildScriptGenerator.Python
 {
-    internal static class ManifestFilePropertyKeys
+    internal static class PythonManifestFilePropertyKeys
     {
         internal const string CompressedVirtualEnvFile = "compressedVirtualEnvFile";
         internal const string VirtualEnvName = "virtualEnvName";

src/BuildScriptGenerator/Python/PythonPlatform.cs

@@ -56,11 +56,12 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
         /// The tar-gz option.
         /// </summary>
         internal const string TarGzOption = "tar-gz";
-
+        private readonly BuildScriptGeneratorOptions _commonOptions;
         private readonly IPythonVersionProvider _pythonVersionProvider;
         private readonly IEnvironment _environment;
         private readonly ILogger<PythonPlatform> _logger;
         private readonly PythonLanguageDetector _detector;
+        private readonly PythonPlatformInstaller _platformInstaller;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="PythonPlatform"/> class.
@@ -71,23 +72,32 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
         /// <param name="logger">The logger of Python platform.</param>
         /// <param name="detector">The detector of Python platform.</param>
         public PythonPlatform(
-            IOptions<PythonScriptGeneratorOptions> pythonScriptGeneratorOptions,
+            IOptions<BuildScriptGeneratorOptions> commonOptions,
             IPythonVersionProvider pythonVersionProvider,
             IEnvironment environment,
             ILogger<PythonPlatform> logger,
-            PythonLanguageDetector detector)
+            PythonLanguageDetector detector,
+            PythonPlatformInstaller platformInstaller)
         {
+            _commonOptions = commonOptions.Value;
             _pythonVersionProvider = pythonVersionProvider;
             _environment = environment;
             _logger = logger;
             _detector = detector;
+            _platformInstaller = platformInstaller;
         }
 
         /// <inheritdoc/>
         public string Name => PythonConstants.PythonName;
 
-        /// <inheritdoc/>
-        public IEnumerable<string> SupportedVersions => _pythonVersionProvider.SupportedPythonVersions;
+        public IEnumerable<string> SupportedVersions
+        {
+            get
+            {
+                var versionInfo = _pythonVersionProvider.GetVersionInfo();
+                return versionInfo.SupportedVersions;
+            }
+        }
 
         /// <inheritdoc/>
         public LanguageDetectorResult Detect(RepositoryContext context)
@@ -98,11 +108,17 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
         /// <inheritdoc/>
         public BuildScriptSnippet GenerateBashBuildScriptSnippet(BuildScriptGeneratorContext context)
         {
+            string installationScriptSnippet = null;
+            if (_commonOptions.EnableDynamicInstall
+                && !_platformInstaller.IsVersionAlreadyInstalled(context.PythonVersion))
+            {
+                installationScriptSnippet = _platformInstaller.GetInstallerScriptSnippet(context.PythonVersion);
+            }
+
             var manifestFileProperties = new Dictionary<string, string>();
 
             // Write the version to the manifest file
-            var key = $"{PythonConstants.PythonName}_version";
-            manifestFileProperties[key] = context.PythonVersion;
+            manifestFileProperties[ManifestFilePropertyKeys.PythonVersion] = context.PythonVersion;
 
             var packageDir = GetPackageDirectory(context);
             var virtualEnvName = GetVirtualEnvironmentName(context);
@@ -122,11 +138,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
                     virtualEnvName = GetDefaultVirtualEnvName(context);
                 }
 
-                manifestFileProperties[ManifestFilePropertyKeys.VirtualEnvName] = virtualEnvName;
+                manifestFileProperties[PythonManifestFilePropertyKeys.VirtualEnvName] = virtualEnvName;
             }
             else
             {
-                manifestFileProperties[ManifestFilePropertyKeys.PackageDir] = packageDir;
+                manifestFileProperties[PythonManifestFilePropertyKeys.PackageDir] = packageDir;
             }
 
             var virtualEnvModule = string.Empty;
@@ -155,7 +171,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
 
             if (!string.IsNullOrWhiteSpace(compressedVirtualEnvFileName))
             {
-                manifestFileProperties[ManifestFilePropertyKeys.CompressedVirtualEnvFile]
+                manifestFileProperties[PythonManifestFilePropertyKeys.CompressedVirtualEnvFile]
                     = compressedVirtualEnvFileName;
             }
 
@@ -178,6 +194,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
             {
                 BashBuildScriptSnippet = script,
                 BuildProperties = manifestFileProperties,
+                PlatformInstallationScriptSnippet = installationScriptSnippet,
             };
         }
 

src/BuildScriptGenerator/Python/PythonPlatformInstaller.cs

@@ -0,0 +1,35 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Python
+{
+    public class PythonPlatformInstaller : PlatformInstallerBase
+    {
+        public PythonPlatformInstaller(
+            IOptions<BuildScriptGeneratorOptions> commonOptions,
+            IEnvironment environment)
+            : base(commonOptions, environment)
+        {
+        }
+
+        public override string GetInstallerScriptSnippet(string version)
+        {
+            return GetInstallerScriptSnippet(PythonConstants.PythonName, version);
+        }
+
+        public override bool IsVersionAlreadyInstalled(string version)
+        {
+            return IsVersionInstalled(
+                version,
+                installationDirs: new[]
+                {
+                    "/opt/python",
+                    $"{Constants.TemporaryInstallationDirectoryRoot}/python"
+                });
+        }
+    }
+}

src/BuildScriptGenerator/Python/PythonScriptGeneratorOptions.cs

@@ -3,19 +3,10 @@
 // Licensed under the MIT license.
 // --------------------------------------------------------------------------------------------
 
-using System.Collections.Generic;
-
 namespace Microsoft.Oryx.BuildScriptGenerator.Python
 {
     public class PythonScriptGeneratorOptions
     {
         public string PythonDefaultVersion { get; set; }
-
-        public string InstalledPythonVersionsDir { get; set; }
-
-        /// <summary>
-        /// Gets or sets the user-provided list of python versions.
-        /// </summary>
-        public IList<string> SupportedPythonVersions { get; set; }
     }
 }

src/BuildScriptGenerator/Python/PythonScriptGeneratorOptionsSetup.cs

@@ -3,7 +3,6 @@
 // Licensed under the MIT license.
 // --------------------------------------------------------------------------------------------
 
-using System.IO;
 using Microsoft.Extensions.Options;
 
 namespace Microsoft.Oryx.BuildScriptGenerator.Python
@@ -26,12 +25,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
             }
 
             options.PythonDefaultVersion = defaultVersion;
-            options.InstalledPythonVersionsDir = PythonConstants.InstalledPythonVersionsDir;
-
-            // Providing the supported versions through an environment variable allows us to use the tool in
-            // other environments, e.g. our local machines for debugging.
-            options.SupportedPythonVersions = _environment.GetEnvironmentVariableAsList(
-                PythonConstants.PythonSupportedVersionsEnvVarName);
         }
     }
 }

src/BuildScriptGenerator/Python/PythonScriptGeneratorServiceCollectionExtensions.cs

@@ -22,6 +22,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator
                 ServiceDescriptor.Singleton<IConfigureOptions<PythonScriptGeneratorOptions>, PythonScriptGeneratorOptionsSetup>());
             services.AddSingleton<IPythonVersionProvider, PythonVersionProvider>();
             services.AddScoped<PythonLanguageDetector>();
+            services.AddScoped<PythonPlatformInstaller>();
+            services.AddSingleton<PythonOnDiskVersionProvider>();
+            services.AddSingleton<PythonSdkStorageVersionProvider>();
             return services;
         }
     }

src/BuildScriptGenerator/Python/PythonVersionProvider.cs

@@ -1,37 +0,0 @@
-// --------------------------------------------------------------------------------------------
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT license.
-// --------------------------------------------------------------------------------------------
-
-using System.Collections.Generic;
-using System.Linq;
-using Microsoft.Extensions.Options;
-
-namespace Microsoft.Oryx.BuildScriptGenerator.Python
-{
-    internal class PythonVersionProvider : IPythonVersionProvider
-    {
-        private readonly PythonScriptGeneratorOptions _options;
-        private IEnumerable<string> _supportedPythonVersions;
-
-        public PythonVersionProvider(IOptions<PythonScriptGeneratorOptions> options)
-        {
-            _options = options.Value;
-        }
-
-        public IEnumerable<string> SupportedPythonVersions
-        {
-            get
-            {
-                if (_supportedPythonVersions == null)
-                {
-                    _supportedPythonVersions = VersionProviderHelper.GetSupportedVersions(
-                        _options.SupportedPythonVersions,
-                        _options.InstalledPythonVersionsDir);
-                }
-
-                return _supportedPythonVersions;
-            }
-        }
-    }
-}

src/BuildScriptGenerator/Python/VersionProviders/IPythonVersionProvider.cs

@@ -3,12 +3,10 @@
 // Licensed under the MIT license.
 // --------------------------------------------------------------------------------------------
 
-using System.Collections.Generic;
-
 namespace Microsoft.Oryx.BuildScriptGenerator.Python
 {
     internal interface IPythonVersionProvider
     {
-        IEnumerable<string> SupportedPythonVersions { get; }
+        PlatformVersionInfo GetVersionInfo();
     }
 }

src/BuildScriptGenerator/Python/VersionProviders/PythonOnDiskVersionProvider.cs

@@ -0,0 +1,35 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Python
+{
+    public class PythonOnDiskVersionProvider : IPythonVersionProvider
+    {
+        private readonly PythonScriptGeneratorOptions _options;
+        private PlatformVersionInfo _platformVersionInfo;
+
+        public PythonOnDiskVersionProvider(IOptions<PythonScriptGeneratorOptions> options)
+        {
+            _options = options.Value;
+        }
+
+        // To enable unit testing
+        public virtual PlatformVersionInfo GetVersionInfo()
+        {
+            if (_platformVersionInfo == null)
+            {
+                var installedVersions = VersionProviderHelper.GetVersionsFromDirectory(
+                            PythonConstants.InstalledPythonVersionsDir);
+                _platformVersionInfo = PlatformVersionInfo.CreateOnDiskVersionInfo(
+                    installedVersions,
+                    _options.PythonDefaultVersion);
+            }
+
+            return _platformVersionInfo;
+        }
+    }
+}

src/BuildScriptGenerator/Python/VersionProviders/PythonSdkStorageVersionProvider.cs

@@ -0,0 +1,25 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System.Net.Http;
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Python
+{
+    internal class PythonSdkStorageVersionProvider : SdkStorageVersionProviderBase, IPythonVersionProvider
+    {
+        public PythonSdkStorageVersionProvider(IEnvironment environment, IHttpClientFactory httpClientFactory)
+            : base(environment, httpClientFactory)
+        {
+        }
+
+        // To enable unit testing
+        public virtual PlatformVersionInfo GetVersionInfo()
+        {
+            return GetAvailableVersionsFromStorage(
+                platformName: "python",
+                versionMetadataElementName: "version");
+        }
+    }
+}

src/BuildScriptGenerator/Python/VersionProviders/PythonVersionProvider.cs

@@ -0,0 +1,36 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using Microsoft.Extensions.Options;
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Python
+{
+    internal class PythonVersionProvider : IPythonVersionProvider
+    {
+        private readonly BuildScriptGeneratorOptions _options;
+        private readonly PythonOnDiskVersionProvider _onDiskVersionProvider;
+        private readonly PythonSdkStorageVersionProvider _sdkStorageVersionProvider;
+
+        public PythonVersionProvider(
+            IOptions<BuildScriptGeneratorOptions> options,
+            PythonOnDiskVersionProvider onDiskVersionProvider,
+            PythonSdkStorageVersionProvider sdkStorageVersionProvider)
+        {
+            _options = options.Value;
+            _onDiskVersionProvider = onDiskVersionProvider;
+            _sdkStorageVersionProvider = sdkStorageVersionProvider;
+        }
+
+        public PlatformVersionInfo GetVersionInfo()
+        {
+            if (_options.EnableDynamicInstall)
+            {
+                return _sdkStorageVersionProvider.GetVersionInfo();
+            }
+
+            return _onDiskVersionProvider.GetVersionInfo();
+        }
+    }
+}

src/BuildScriptGenerator/SdkStorageVersionProviderBase.cs

@@ -0,0 +1,66 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Xml.Linq;
+using System.Xml.XPath;
+using Microsoft.Oryx.Common;
+
+namespace Microsoft.Oryx.BuildScriptGenerator
+{
+    public class SdkStorageVersionProviderBase
+    {
+        private readonly IEnvironment _environment;
+        private readonly IHttpClientFactory _httpClientFactory;
+
+        public SdkStorageVersionProviderBase(IEnvironment environment, IHttpClientFactory httpClientFactory)
+        {
+            _environment = environment;
+            _httpClientFactory = httpClientFactory;
+        }
+
+        protected PlatformVersionInfo GetAvailableVersionsFromStorage(
+            string platformName,
+            string versionMetadataElementName)
+        {
+            var httpClient = _httpClientFactory.CreateClient("general");
+
+            var sdkStorageBaseUrl = GetPlatformBinariesStorageBaseUrl();
+            var blobList = httpClient
+                .GetStringAsync($"{sdkStorageBaseUrl}/{platformName}?restype=container&comp=list&include=metadata")
+                .Result;
+            var xdoc = XDocument.Parse(blobList);
+            var supportedVersions = new List<string>();
+            foreach (var runtimeVersionElement in xdoc.XPathSelectElements(
+                $"//Blobs/Blob/Metadata/{versionMetadataElementName}"))
+            {
+                supportedVersions.Add(runtimeVersionElement.Value);
+            }
+
+            var defaultVersion = httpClient
+                .GetStringAsync($"{sdkStorageBaseUrl}/{platformName}/default_version.txt")
+                .Result;
+
+            return PlatformVersionInfo.CreateAvailableOnWebVersionInfo(supportedVersions, defaultVersion);
+        }
+
+        private string GetPlatformBinariesStorageBaseUrl()
+        {
+            var platformBinariesStorageBaseUrl = _environment.GetEnvironmentVariable(
+                SdkStorageConstants.SdkStorageBaseUrlKeyName);
+            if (string.IsNullOrEmpty(platformBinariesStorageBaseUrl))
+            {
+                throw new InvalidOperationException(
+                    $"Environment variable '{SdkStorageConstants.SdkStorageBaseUrlKeyName}' is required.");
+            }
+
+            platformBinariesStorageBaseUrl = platformBinariesStorageBaseUrl.TrimEnd('/');
+            return platformBinariesStorageBaseUrl;
+        }
+    }
+}

src/BuildScriptGeneratorCli/Commands/BuildCommand.cs

@@ -15,7 +15,6 @@ using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.Oryx.BuildScriptGenerator;
-using Microsoft.Oryx.BuildScriptGenerator.Exceptions;
 using Microsoft.Oryx.Common;
 
 namespace Microsoft.Oryx.BuildScriptGeneratorCli
@@ -364,6 +363,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
                 shouldPackage: ShouldPackage,
                 requiredOsPackages: string.IsNullOrWhiteSpace(OsRequirements) ? null : OsRequirements.Split(','),
                 scriptOnly: false,
+                enableDynamicInstall: EnableDynamicInstall,
                 properties: Properties);
         }
 

src/BuildScriptGeneratorCli/Commands/BuildCommandBase.cs

@@ -38,5 +38,9 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
             CommandOptionType.MultipleValue,
             Description = "Additional information used by this tool to generate and run build scripts.")]
         public string[] Properties { get; set; }
+
+        [Option("--enable-dynamic-install", CommandOptionType.NoValue,
+            Description = "Enables dynamic installation of platform versions when a version is not found on the disk.")]
+        public bool EnableDynamicInstall { get; set; }
     }
 }

src/BuildScriptGeneratorCli/Commands/BuildScriptCommand.cs

@@ -93,6 +93,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
                 shouldPackage: ShouldPackage,
                 requiredOsPackages: string.IsNullOrWhiteSpace(OsRequirements) ? null : OsRequirements.Split(','),
                 scriptOnly: true,
+                enableDynamicInstall: EnableDynamicInstall,
                 properties: Properties);
         }
     }

src/BuildScriptGeneratorCli/Commands/BuildScriptGeneratorOptionsHelper.cs

@@ -24,6 +24,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
             bool shouldPackage,
             string[] requiredOsPackages,
             bool scriptOnly,
+            bool enableDynamicInstall,
             string[] properties)
         {
             options.SourceDir = string.IsNullOrEmpty(sourceDir)
@@ -52,6 +53,8 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
 
             options.ScriptOnly = scriptOnly;
 
+            options.EnableDynamicInstall = enableDynamicInstall;
+
             // Process properties
             if (properties != null)
             {

src/BuildScriptGeneratorCli/Commands/BuildpackDetectCommand.cs

@@ -85,6 +85,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
                 shouldPackage: false,
                 requiredOsPackages: null,
                 scriptOnly: false,
+                enableDynamicInstall: false,
                 properties: null);
         }
 

src/BuildScriptGeneratorCli/Commands/ExecCommand.cs

@@ -100,6 +100,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli
                 shouldPackage: false,
                 requiredOsPackages: null,
                 scriptOnly: false,
+                enableDynamicInstall: false,
                 null);
         }
     }

src/Common/SdkStorageConstants.cs

@@ -0,0 +1,11 @@
+// This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+namespace Microsoft.Oryx.Common
+{
+    public static class SdkStorageConstants
+    {
+        public const string SdkStorageBaseUrlKeyName = "ORYX_SDK_STORAGE_BASE_URL";
+        public const string DevSdkStorageBaseUrl = "https://oryxsdksdev.blob.core.windows.net";
+        public const string ProdSdkStorageBaseUrl = "https://oryxsdks.blob.core.windows.net";
+    }
+}

src/startupscriptgenerator/src/common/buildManifestHelper.go

@@ -24,6 +24,10 @@ type BuildManifest struct {
 	StartupDllFileName        string
 	InjectedAppInsights       string
 	CompressedNodeModulesFile string
+	DotNetCoreSdkVersion      string
+	DotNetCoreRuntimeVersion  string
+	NodeVersion               string
+	PythonVersion             string
 }
 
 var _buildManifest BuildManifest

src/startupscriptgenerator/src/common/commandLineFlags.go

@@ -5,10 +5,39 @@
 
 package common
 
-import "flag"
+import (
+	"common/consts"
+	"flag"
+	"fmt"
+	"os"
+)
 
-var ManifestDirFlag = flag.String(
-	"manifestDir",
-	"",
-	"[Optional] Path to the directory where build manifest file can be found. If no value is provided, then "+
-		"it is assumed to be under the directory specified by 'appPath'.")
+func ValidateCommands(versionCommand *flag.FlagSet, scriptCommand *flag.FlagSet, setupEnvCommand *flag.FlagSet) {
+	// Verify that a subcommand has been provided
+	// os.Arg[0] is the main command
+	// os.Arg[1] will be the subcommand
+	if len(os.Args) < 2 {
+		fmt.Println(fmt.Sprintf(
+			"Error: '%s' or '%s' or '%s' subcommand is required",
+			consts.VersionCommandName,
+			consts.SetupEnvCommandName,
+			consts.CreateScriptCommandName))
+		os.Exit(1)
+	}
+
+	// Switch on the subcommand
+	// Parse the flags for appropriate FlagSet
+	// FlagSet.Parse() requires a set of arguments to parse as input
+	// os.Args[2:] will be all arguments starting after the subcommand at os.Args[1]
+	switch os.Args[1] {
+	case consts.VersionCommandName:
+		PrintVersionInfo()
+	case consts.CreateScriptCommandName:
+		scriptCommand.Parse(os.Args[2:])
+	case consts.SetupEnvCommandName:
+		setupEnvCommand.Parse(os.Args[2:])
+	default:
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+}

src/startupscriptgenerator/src/common/consts/commands.go

@@ -0,0 +1,10 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+package consts
+
+const SetupEnvCommandName string = "setupEnv"
+const CreateScriptCommandName string = "create-script"
+const VersionCommandName string = "version"

src/startupscriptgenerator/src/common/consts/runtime_dynamic_installation_constants.go

@@ -0,0 +1,8 @@
+// This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+package consts
+
+const NodeInstallationDir string = "/tmp/oryx/nodejs"
+const DotNetCoreInstallationDir string = "/tmp/oryx/dotnet"
+const PythonInstallationRootDir string = "/opt/python"
+const SetupScriptLocation string = "/tmp/oryx/setupEnv.sh"

src/startupscriptgenerator/src/common/consts/sdk_storage_constants.go

@@ -0,0 +1,7 @@
+// This file was auto-generated from 'constants.yaml'. Changes may be overridden.
+
+package consts
+
+const SdkStorageBaseUrlKeyName string = "ORYX_SDK_STORAGE_BASE_URL"
+const DevSdkStorageBaseUrl string = "https://oryxsdksdev.blob.core.windows.net"
+const ProdSdkStorageBaseUrl string = "https://oryxsdks.blob.core.windows.net"

src/startupscriptgenerator/src/common/setupEnvironment.go

@@ -0,0 +1,67 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+package common
+
+import (
+	"bytes"
+	"common/consts"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func SetupEnv(script string) {
+	WriteScript(consts.SetupScriptLocation, script)
+	cmd := exec.Command("/bin/sh", consts.SetupScriptLocation)
+	var outb, errb bytes.Buffer
+	cmd.Stdout = &outb
+	cmd.Stderr = &errb
+	err := cmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(outb.String(), errb.String())
+}
+
+func GetSetupScript(platformName string, version string, installationDir string) string {
+	sdkStorageBaseUrl := os.Getenv(consts.SdkStorageBaseUrlKeyName)
+	if sdkStorageBaseUrl == "" {
+		panic("Environment variable " + consts.SdkStorageBaseUrlKeyName + " is required.")
+	}
+
+	tarFile := fmt.Sprintf("%s.tar.gz", version)
+	scriptBuilder := strings.Builder{}
+	scriptBuilder.WriteString("#!/bin/sh\n")
+	scriptBuilder.WriteString("set -e\n")
+	scriptBuilder.WriteString("echo\n")
+	scriptBuilder.WriteString(
+		fmt.Sprintf("echo Downloading and installing '%s' version '%s'...\n", platformName, version))
+	scriptBuilder.WriteString(fmt.Sprintf("rm -rf %s\n", installationDir))
+	scriptBuilder.WriteString(fmt.Sprintf("mkdir -p %s\n", installationDir))
+	scriptBuilder.WriteString(fmt.Sprintf("cd %s\n", installationDir))
+	scriptBuilder.WriteString(
+		fmt.Sprintf("curl -D headers.txt -SL \"%s/%s/%s-%s.tar.gz\" --output %s >/dev/null 2>&1\n",
+			sdkStorageBaseUrl,
+			platformName,
+			platformName,
+			version,
+			tarFile))
+	scriptBuilder.WriteString("headerName=\"x-ms-meta-checksum\"\n")
+	scriptBuilder.WriteString(fmt.Sprintf(
+		"checksumHeader=$(cat headers.txt | grep $headerName: | tr -d '%s')\n",
+		"\\r"))
+	scriptBuilder.WriteString("rm -f headers.txt\n")
+	scriptBuilder.WriteString("checksumValue=${checksumHeader#\"$headerName: \"}\n")
+	scriptBuilder.WriteString(fmt.Sprintf("echo \"$checksumValue %s.tar.gz\" | sha512sum -c - >/dev/null 2>&1\n", version))
+	scriptBuilder.WriteString(fmt.Sprintf("tar -xzf %s -C .\n", tarFile))
+	scriptBuilder.WriteString(fmt.Sprintf("rm -f %s\n", tarFile))
+	scriptBuilder.WriteString(fmt.Sprintf("echo Done. Installed at '%s'\n", installationDir))
+	scriptBuilder.WriteString(fmt.Sprintf("rm -f %s\n", tarFile))
+	scriptBuilder.WriteString("echo\n")
+	return scriptBuilder.String()
+}

src/startupscriptgenerator/src/dotnetcore/main.go

@@ -28,7 +28,11 @@ func main() {
 		"runFromPath",
 		"",
 		"The path to the directory where the output is copied and run from there.")
-	manifestDirPtr := common.ManifestDirFlag
+	manifestDirPtr := flag.String(
+		"manifestDir",
+		"",
+		"[Optional] Path to the directory where build manifest file can be found. If no value is provided, then "+
+			"it is assumed to be under the directory specified by 'appPath'.")
 	bindPortPtr := flag.String("bindPort", "", "[Optional] Port where the application will bind to. Default is 8080")
 	userStartupCommandPtr := flag.String(
 		"userStartupCommand",

src/startupscriptgenerator/src/node/main.go

@@ -17,7 +17,11 @@ func main() {
 	common.PrintVersionInfo()
 
 	appPathPtr := flag.String("appPath", ".", "The path to the application folder, e.g. '/home/site/wwwroot/'.")
-	manifestDirPtr := common.ManifestDirFlag
+	manifestDirPtr := flag.String(
+		"manifestDir",
+		"",
+		"[Optional] Path to the directory where build manifest file can be found. If no value is provided, then "+
+			"it is assumed to be under the directory specified by 'appPath'.")
 	userStartupCommandPtr := flag.String("userStartupCommand", "", "[Optional] Command that will be executed to start the application up.")
 	defaultAppFilePathPtr := flag.String("defaultApp", "", "[Optional] Path to a default file that will be executed if the entrypoint is not found. Ex: '/opt/startup/default-static-site.js'")
 	bindPortPtr := flag.String("bindPort", "", "[Optional] Port where the application will bind to. Default is 8080")

src/startupscriptgenerator/src/php/main.go

@@ -15,7 +15,11 @@ import (
 func main() {
 	common.PrintVersionInfo()
 	appPathPtr := flag.String("appPath", ".", "The path to the application folder, e.g. '/home/site/wwwroot/'.")
-	manifestDirPtr := common.ManifestDirFlag
+	manifestDirPtr := flag.String(
+		"manifestDir",
+		"",
+		"[Optional] Path to the directory where build manifest file can be found. If no value is provided, then "+
+			"it is assumed to be under the directory specified by 'appPath'.")
 
 	var _phpOrigin = os.Getenv(consts.PhpOriginEnvVarName)
 	var startupCommand = ""

src/startupscriptgenerator/src/python/main.go

@@ -7,79 +7,119 @@ package main
 
 import (
 	"common"
+	"common/consts"
 	"flag"
+	"fmt"
+	"strings"
 )
 
 func main() {
-	common.PrintVersionInfo()
+	versionCommand := flag.NewFlagSet(consts.VersionCommandName, flag.ExitOnError)
 
-	appPathPtr := flag.String("appPath", ".", "The path to the application folder, e.g. '/home/site/wwwroot/'.")
+	setupEnvCommand := flag.NewFlagSet(consts.SetupEnvCommandName, flag.ExitOnError)
+	setupEnvAppPathPtr := setupEnvCommand.String("appPath", ".", "The path to the application folder, e.g. '/home/site/wwwroot/'.")
 
-	manifestDirPtr := common.ManifestDirFlag
-
-	userStartupCommandPtr := flag.String("userStartupCommand", "", "[Optional] Command that will be executed "+
+	scriptCommand := flag.NewFlagSet(consts.CreateScriptCommandName, flag.ExitOnError)
+	appPathPtr := scriptCommand.String("appPath", ".", "The path to the application folder, e.g. '/home/site/wwwroot/'.")
+	manifestDirPtr := scriptCommand.String(
+		"manifestDir",
+		"",
+		"[Optional] Path to the directory where build manifest file can be found. If no value is provided, then "+
+			"it is assumed to be under the directory specified by 'appPath'.")
+	userStartupCommandPtr := scriptCommand.String("userStartupCommand", "", "[Optional] Command that will be executed "+
 		"to start the application up.")
-
-	defaultAppFilePathPtr := flag.String("defaultApp", "", "[Optional] Path to a default file that will be "+
+	defaultAppFilePathPtr := scriptCommand.String("defaultApp", "", "[Optional] Path to a default file that will be "+
 		"executed if the entrypoint is not found. Ex: '/opt/defaultsite'")
-
-	defaultAppModulePtr := flag.String("defaultAppModule", "application:app", "Module of the default application,"+
+	defaultAppModulePtr := scriptCommand.String("defaultAppModule", "application:app", "Module of the default application,"+
 		" e.g. 'application:app'.")
-
-	defaultAppDebugModulePtr := flag.String("defaultAppDebugModule", "application.py", "Module to run if "+
+	defaultAppDebugModulePtr := scriptCommand.String("defaultAppDebugModule", "application.py", "Module to run if "+
 		"running the app in debug mode, e.g. 'application.py start_dev_server'. Has no effect if -debugAdapter isn't used.")
-
-	debugAdapterPtr := flag.String("debugAdapter", "", "Python debugger adapter. Currently, only 'ptvsd' is "+
+	debugAdapterPtr := scriptCommand.String("debugAdapter", "", "Python debugger adapter. Currently, only 'ptvsd' is "+
 		"supported.")
-
-	debugPortPtr := flag.String("debugPort", "5678", "Port where the debugger will bind to. Has no effect if -debugAdapter isn't used.")
-
-	debugWaitPtr := flag.Bool("debugWait", false, "Whether the debugger adapter should pause and wait for a "+
+	debugPortPtr := scriptCommand.String("debugPort", "5678", "Port where the debugger will bind to. Has no effect if -debugAdapter isn't used.")
+	debugWaitPtr := scriptCommand.Bool("debugWait", false, "Whether the debugger adapter should pause and wait for a "+
 		"client connection before running the app.")
-
-	virtualEnvNamePtr := flag.String("virtualEnvName", "", "Name of the virtual environment for the app")
-
-	packagesFolderPtr := flag.String("packagedir", "", "Directory where the python packages were installed, if "+
+	virtualEnvNamePtr := scriptCommand.String("virtualEnvName", "", "Name of the virtual environment for the app")
+	packagesFolderPtr := scriptCommand.String("packagedir", "", "Directory where the python packages were installed, if "+
 		"no virtual environment was used.")
-
-	bindPortPtr := flag.String("bindPort", "", "[Optional] Port where the application will bind to. Default is 80")
-
-	outputPathPtr := flag.String("output", "run.sh", "Path to the script to be generated.")
-
-	skipVirtualEnvExtraction := flag.Bool(
+	bindPortPtr := scriptCommand.String("bindPort", "", "[Optional] Port where the application will bind to. Default is 80")
+	outputPathPtr := scriptCommand.String("output", "run.sh", "Path to the script to be generated.")
+	skipVirtualEnvExtraction := scriptCommand.Bool(
 		"skipVirtualEnvExtraction",
 		false,
 		"Disables the extraction of the compressed virtual environment file. If used, some external tool will "+
 			"have to extract it - otherwise the application might not work.")
 
-	flag.Parse()
-
 	logger := common.GetLogger("python.main")
 	defer logger.Shutdown()
 	logger.StartupScriptRequested()
 
-	fullAppPath := common.GetValidatedFullPath(*appPathPtr)
-	defaultAppFullPath := common.GetValidatedFullPath(*defaultAppFilePathPtr)
+	common.ValidateCommands(versionCommand, scriptCommand, setupEnvCommand)
 
-	buildManifest := common.GetBuildManifest(manifestDirPtr, fullAppPath)
-	common.SetGlobalOperationID(buildManifest)
+	if scriptCommand.Parsed() {
+		fullAppPath := common.GetValidatedFullPath(*appPathPtr)
+		defaultAppFullPath := common.GetValidatedFullPath(*defaultAppFilePathPtr)
 
-	entrypointGenerator := PythonStartupScriptGenerator{
-		AppPath:                  fullAppPath,
-		UserStartupCommand:       *userStartupCommandPtr,
-		VirtualEnvName:           *virtualEnvNamePtr,
-		BindPort:                 *bindPortPtr,
-		DefaultAppPath:           defaultAppFullPath,
-		DefaultAppModule:         *defaultAppModulePtr,
-		DefaultAppDebugModule:    *defaultAppDebugModulePtr,
-		DebugAdapter:             *debugAdapterPtr,
-		DebugPort:                *debugPortPtr,
-		DebugWait:                *debugWaitPtr,
-		PackageDirectory:         *packagesFolderPtr,
-		SkipVirtualEnvExtraction: *skipVirtualEnvExtraction,
-		Manifest:                 buildManifest,
+		buildManifest := common.GetBuildManifest(manifestDirPtr, fullAppPath)
+		common.SetGlobalOperationID(buildManifest)
+
+		entrypointGenerator := PythonStartupScriptGenerator{
+			AppPath:                  fullAppPath,
+			UserStartupCommand:       *userStartupCommandPtr,
+			VirtualEnvName:           *virtualEnvNamePtr,
+			BindPort:                 *bindPortPtr,
+			DefaultAppPath:           defaultAppFullPath,
+			DefaultAppModule:         *defaultAppModulePtr,
+			DefaultAppDebugModule:    *defaultAppDebugModulePtr,
+			DebugAdapter:             *debugAdapterPtr,
+			DebugPort:                *debugPortPtr,
+			DebugWait:                *debugWaitPtr,
+			PackageDirectory:         *packagesFolderPtr,
+			SkipVirtualEnvExtraction: *skipVirtualEnvExtraction,
+			Manifest:                 buildManifest,
+		}
+
+		command := entrypointGenerator.GenerateEntrypointScript()
+		common.WriteScript(*outputPathPtr, command)
 	}
 
-	command := entrypointGenerator.GenerateEntrypointScript()
-	common.WriteScript(*outputPathPtr, command)
+	if setupEnvCommand.Parsed() {
+		fullAppPath := common.GetValidatedFullPath(*setupEnvAppPathPtr)
+		buildManifest := common.GetBuildManifest(manifestDirPtr, fullAppPath)
+		pythonInstallationRoot := fmt.Sprintf(
+			"%s/%s",
+			consts.PythonInstallationRootDir,
+			buildManifest.PythonVersion)
+		script := common.GetSetupScript(
+			"python",
+			buildManifest.PythonVersion,
+			pythonInstallationRoot)
+		scriptBuilder := strings.Builder{}
+		scriptBuilder.WriteString(script)
+		scriptBuilder.WriteString(
+			fmt.Sprintf("echo %s/lib > /etc/ld.so.conf.d/python.conf\n", pythonInstallationRoot))
+		scriptBuilder.WriteString("ldconfig\n")
+
+		if strings.HasPrefix(buildManifest.PythonVersion, "3.") {
+			scriptBuilder.WriteString(
+				fmt.Sprintf("cd %s/bin\n", pythonInstallationRoot))
+			scriptBuilder.WriteString("rm -f python\n")
+			scriptBuilder.WriteString("ln -s python3 python\n")
+			scriptBuilder.WriteString("ln -s idle3 idle\n")
+			scriptBuilder.WriteString("ln -s pydoc3 pydoc\n")
+			scriptBuilder.WriteString("ln -s python3-config python-config\n")
+		}
+		// To enable following pip commands to run, set the path env variable
+		scriptBuilder.WriteString(
+			fmt.Sprintf("export PATH=\"%s/bin:${PATH}\"\n", pythonInstallationRoot))
+		scriptBuilder.WriteString("pip install --upgrade pip\n")
+		scriptBuilder.WriteString("pip install gunicorn\n")
+		scriptBuilder.WriteString("pip install ptvsd\n")
+		// To enable future interactive or non-interactive shells, write to bashrc file
+		finalScript := scriptBuilder.String()
+		fmt.Println(fmt.Sprintf(
+			"Setting up the environment with 'python' version '%s'...\n",
+			buildManifest.PythonVersion))
+		common.SetupEnv(finalScript)
+	}
 }

src/startupscriptgenerator/src/python/scriptgenerator.go

@@ -43,14 +43,23 @@ func (gen *PythonStartupScriptGenerator) GenerateEntrypointScript() string {
 
 	logger.LogInformation("Generating script for source.")
 
+	pythonInstallationRoot := fmt.Sprintf("/opt/python/%s", gen.Manifest.PythonVersion)
+
+	common.PrintVersionInfo()
+
 	scriptBuilder := strings.Builder{}
-	scriptBuilder.WriteString("#!/bin/sh\n")
+	scriptBuilder.WriteString("#!/bin/sh\n\n")
+	if !common.PathExists(pythonInstallationRoot) {
+		scriptBuilder.WriteString(fmt.Sprintf("oryx setupEnv -appPath %s\n", gen.AppPath))
+	}
 	scriptBuilder.WriteString("\n# Enter the source directory to make sure the script runs where the user expects\n")
 	scriptBuilder.WriteString("cd " + gen.AppPath + "\n\n")
 
 	common.SetEnvironmentVariableInScript(&scriptBuilder, "HOST", "", DefaultHost)
 	common.SetEnvironmentVariableInScript(&scriptBuilder, "PORT", gen.BindPort, DefaultBindPort)
 
+	scriptBuilder.WriteString(fmt.Sprintf("export PATH=\"%s/bin:${PATH}\"\n", pythonInstallationRoot))
+
 	packageSetupBlock := gen.getPackageSetupCommand()
 	scriptBuilder.WriteString(packageSetupBlock)
 

tests/BuildScriptGenerator.Tests/Python/PythonLanguageDetectorTest.cs

@@ -3,9 +3,7 @@
 // Licensed under the MIT license.
 // --------------------------------------------------------------------------------------------
 
-using System.Collections.Generic;
 using Microsoft.Extensions.Logging.Abstractions;
-using Microsoft.Extensions.Options;
 using Microsoft.Oryx.BuildScriptGenerator.Exceptions;
 using Microsoft.Oryx.BuildScriptGenerator.Python;
 using Microsoft.Oryx.Tests.Common;
@@ -26,8 +24,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
         public void Detect_ReturnsNull_WhenSourceDirectoryIsEmpty()
         {
             // Arrange
+            var version = "100.100.100";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { version },
+                defaultVersion: version);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             // No files in source directory
             var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
@@ -44,8 +44,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
         public void Detect_ReutrnsNull_WhenRequirementsFileDoesNotExist()
         {
             // Arrange
+            var version = "100.100.100";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { version },
+                defaultVersion: version);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             IOHelpers.CreateFile(sourceDir, "foo.py content", "foo.py");
             var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
@@ -62,8 +64,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
         public void Detect_ReutrnsNull_WhenRequirementsTextFileExists_ButNoPyOrRuntimeFileExists()
         {
             // Arrange
+            var version = "100.100.100";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { version },
+                defaultVersion: version);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             // No files with '.py' or no runtime.txt file
             IOHelpers.CreateFile(sourceDir, "requirements.txt content", PythonConstants.RequirementsFileName);
@@ -81,12 +85,15 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
         public void Detect_ReutrnsResult_WhenNoPyFileExists_ButRuntimeTextFileExists_HavingPythonVersionInIt()
         {
             // Arrange
+            var expectedVersion = "1000.1000.1000";
+            var defaultVersion = "1000.1000.1001";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { defaultVersion, expectedVersion },
+                defaultVersion: defaultVersion);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             // No file with a '.py' extension
             IOHelpers.CreateFile(sourceDir, "", PythonConstants.RequirementsFileName);
-            IOHelpers.CreateFile(sourceDir, $"python-{Common.PythonVersions.Python37Version}", "runtime.txt");
+            IOHelpers.CreateFile(sourceDir, $"python-{expectedVersion}", "runtime.txt");
             var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
             var context = CreateContext(repo);
 
@@ -96,27 +103,29 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             // Assert
             Assert.NotNull(result);
             Assert.Equal("python", result.Language);
-            Assert.Equal(Common.PythonVersions.Python37Version, result.LanguageVersion);
+            Assert.Equal(expectedVersion, result.LanguageVersion);
         }
 
         [Fact]
         public void Detect_Throws_WhenUnsupportedPythonVersion_FoundInRuntimeFile()
         {
             // Arrange
-            var badVersion = "100.100.100";
+            var unsupportedVersion = "100.100.100";
+            var supportedVersion = "1.2.3";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { supportedVersion },
+                defaultVersion: supportedVersion);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             IOHelpers.CreateFile(sourceDir, "", PythonConstants.RequirementsFileName);
-            IOHelpers.CreateFile(sourceDir, "python-" + badVersion, PythonConstants.RuntimeFileName);
+            IOHelpers.CreateFile(sourceDir, "python-" + unsupportedVersion, PythonConstants.RuntimeFileName);
             var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
             var context = CreateContext(repo);
 
             // Act & Assert
             var exception = Assert.Throws<UnsupportedVersionException>(() => detector.Detect(context));
             Assert.Equal(
-                $"Platform 'python' version '{badVersion}' is unsupported. " +
-                $"Supported versions: {Common.PythonVersions.Python37Version}",
+                $"Platform 'python' version '{unsupportedVersion}' is unsupported. " +
+                $"Supported versions: {supportedVersion}",
                 exception.Message);
         }
 
@@ -127,8 +136,10 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
         public void Detect_ReutrnsNull_WhenRuntimeTextFileExists_ButDoesNotTextInExpectedFormat(string fileContent)
         {
             // Arrange
+            var supportedVersion = "1.2.3";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { supportedVersion },
+                defaultVersion: supportedVersion);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             IOHelpers.CreateFile(sourceDir, "", PythonConstants.RequirementsFileName);
             IOHelpers.CreateFile(sourceDir, fileContent, "runtime.txt");
@@ -142,12 +153,62 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             Assert.Null(result);
         }
 
+        [Fact]
+        public void Detect_ReturnsResult_WhenOnlyMajorVersion_IsSpecifiedInRuntimeTxtFile()
+        {
+            // Arrange
+            var runtimeTxtVersion = "1";
+            var expectedVersion = "1.2.3";
+            var detector = CreatePythonLanguageDetector(
+                supportedPythonVersions: new[] { "100.100.100", "1.2.1", expectedVersion },
+                defaultVersion: expectedVersion);
+            var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
+            IOHelpers.CreateFile(sourceDir, "", PythonConstants.RequirementsFileName);
+            IOHelpers.CreateFile(sourceDir, $"python-{runtimeTxtVersion}", PythonConstants.RuntimeFileName);
+            var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
+            var context = CreateContext(repo);
+
+            // Act
+            var result = detector.Detect(context);
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.Equal("python", result.Language);
+            Assert.Equal(expectedVersion, result.LanguageVersion);
+        }
+
+        [Fact]
+        public void Detect_ReturnsResult_WhenOnlyMajorAndMinorVersion_AreSpecifiedInRuntimeTxtFile()
+        {
+            // Arrange
+            var runtimeTxtVersion = "1.2";
+            var expectedVersion = "1.2.3";
+            var detector = CreatePythonLanguageDetector(
+                supportedPythonVersions: new[] { "100.100.100", "1.2.1r", expectedVersion },
+                defaultVersion: expectedVersion);
+            var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
+            IOHelpers.CreateFile(sourceDir, "", PythonConstants.RequirementsFileName);
+            IOHelpers.CreateFile(sourceDir, $"python-{runtimeTxtVersion}", PythonConstants.RuntimeFileName);
+            var repo = new LocalSourceRepo(sourceDir, NullLoggerFactory.Instance);
+            var context = CreateContext(repo);
+
+            // Act
+            var result = detector.Detect(context);
+
+            // Assert
+            Assert.NotNull(result);
+            Assert.Equal("python", result.Language);
+            Assert.Equal(expectedVersion, result.LanguageVersion);
+        }
+
         [Fact]
         public void Detect_ReturnsResult_WithPythonDefaultVersion_WhenNoRuntimeTextFileExists()
         {
             // Arrange
+            var expectedVersion = "1.2.3";
             var detector = CreatePythonLanguageDetector(
-                supportedPythonVersions: new[] { Common.PythonVersions.Python37Version });
+                supportedPythonVersions: new[] { "100.100.100", expectedVersion },
+                defaultVersion: expectedVersion);
             var sourceDir = IOHelpers.CreateTempDir(_tempDirRoot);
             IOHelpers.CreateFile(sourceDir, "content", PythonConstants.RequirementsFileName);
             IOHelpers.CreateFile(sourceDir, "foo.py content", "foo.py");
@@ -160,7 +221,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             // Assert
             Assert.NotNull(result);
             Assert.Equal("python", result.Language);
-            Assert.Equal(Common.PythonVersions.Python37Version, result.LanguageVersion);
+            Assert.Equal(expectedVersion, result.LanguageVersion);
         }
 
         private BuildScriptGeneratorContext CreateContext(ISourceRepo sourceRepo)
@@ -171,13 +232,15 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             };
         }
 
-        private PythonLanguageDetector CreatePythonLanguageDetector(string[] supportedPythonVersions)
+        private PythonLanguageDetector CreatePythonLanguageDetector(
+            string[] supportedPythonVersions, string defaultVersion)
         {
-            return CreatePythonLanguageDetector(supportedPythonVersions, new TestEnvironment());
+            return CreatePythonLanguageDetector(supportedPythonVersions, defaultVersion, new TestEnvironment());
         }
 
         private PythonLanguageDetector CreatePythonLanguageDetector(
             string[] supportedPythonVersions,
+            string defaultVersion,
             IEnvironment environment)
         {
             var optionsSetup = new PythonScriptGeneratorOptionsSetup(environment);
@@ -185,20 +248,26 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             optionsSetup.Configure(options);
 
             return new PythonLanguageDetector(
-                Options.Create(options),
-                new TestPythonVersionProvider(supportedPythonVersions),
+                new TestPythonVersionProvider(supportedPythonVersions, defaultVersion),
                 NullLogger<PythonLanguageDetector>.Instance,
                 new DefaultStandardOutputWriter());
         }
 
         private class TestPythonVersionProvider : IPythonVersionProvider
         {
-            public TestPythonVersionProvider(string[] supportedPythonVersions)
+            private readonly string[] _supportedPythonVersions;
+            private readonly string _defaultVersion;
+
+            public TestPythonVersionProvider(string[] supportedPythonVersions, string defaultVersion)
             {
-                SupportedPythonVersions = supportedPythonVersions;
+                _supportedPythonVersions = supportedPythonVersions;
+                _defaultVersion = defaultVersion;
             }
 
-            public IEnumerable<string> SupportedPythonVersions { get; }
+            public PlatformVersionInfo GetVersionInfo()
+            {
+                return PlatformVersionInfo.CreateOnDiskVersionInfo(_supportedPythonVersions, _defaultVersion);
+            }
         }
     }
 }

tests/BuildScriptGenerator.Tests/Python/PythonPlatformTests.cs

@@ -14,11 +14,99 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
 {
     public class PythonPlatformTests
     {
+        [Fact]
+        public void GeneratedSnippet_DoesNotHaveInstallScript_IfDynamicInstallIsDisabled()
+        {
+            // Arrange
+            var options = new BuildScriptGeneratorOptions() { EnableDynamicInstall = false };
+            var environment = new TestEnvironment();
+            var installerScriptSnippet = "##INSTALLER_SCRIPT##";
+            var versionProvider = new TestPythonVersionProvider(new[] { "3.7.5", "3.8.0" }, defaultVersion: "3.7.5");
+            var platformInstaller = new TestPythonPlatformInstaller(
+                isVersionAlreadyInstalled: false,
+                installerScript: installerScriptSnippet,
+                Options.Create(options),
+                environment);
+            var platform = CreatePlatform(environment, versionProvider, platformInstaller, options);
+            var repo = new MemorySourceRepo();
+            repo.AddFile("", PythonConstants.RequirementsFileName);
+            repo.AddFile("print(1)", "bla.py");
+            var context = new BuildScriptGeneratorContext { SourceRepo = repo, PythonVersion = "3.7.5" };
+
+            // Act
+            var snippet = platform.GenerateBashBuildScriptSnippet(context);
+
+            // Assert
+            Assert.NotNull(snippet);
+            Assert.Null(snippet.PlatformInstallationScriptSnippet);
+            Assert.Contains(ManifestFilePropertyKeys.PythonVersion, snippet.BuildProperties.Keys);
+            Assert.Equal("3.7.5", snippet.BuildProperties[ManifestFilePropertyKeys.PythonVersion]);
+        }
+
+        [Fact]
+        public void GeneratedSnippet_HasInstallationScript_IfDynamicInstallIsEnabled()
+        {
+            // Arrange
+            var options = new BuildScriptGeneratorOptions() { EnableDynamicInstall = true };
+            var environment = new TestEnvironment();
+            var installerScriptSnippet = "##INSTALLER_SCRIPT##";
+            var versionProvider = new TestPythonVersionProvider(new[] { "3.7.5", "3.8.0" }, defaultVersion: "3.7.5");
+            var platformInstaller = new TestPythonPlatformInstaller(
+                isVersionAlreadyInstalled: false,
+                installerScript: installerScriptSnippet,
+                Options.Create(options),
+                environment);
+            var platform = CreatePlatform(environment, versionProvider, platformInstaller, options);
+            var repo = new MemorySourceRepo();
+            repo.AddFile("", PythonConstants.RequirementsFileName);
+            repo.AddFile("print(1)", "bla.py");
+            var context = new BuildScriptGeneratorContext { SourceRepo = repo, PythonVersion = "3.7.5" };
+
+            // Act
+            var snippet = platform.GenerateBashBuildScriptSnippet(context);
+
+            // Assert
+            Assert.NotNull(snippet);
+            Assert.NotNull(snippet.PlatformInstallationScriptSnippet);
+            Assert.Equal(installerScriptSnippet, snippet.PlatformInstallationScriptSnippet);
+            Assert.Contains(ManifestFilePropertyKeys.PythonVersion, snippet.BuildProperties.Keys);
+            Assert.Equal("3.7.5", snippet.BuildProperties[ManifestFilePropertyKeys.PythonVersion]);
+        }
+
+        [Fact]
+        public void GeneratedSnippet_DoesNotHaveInstallScript_IfVersionIsAlreadyPresentOnDisk()
+        {
+            // Arrange
+            var options = new BuildScriptGeneratorOptions() { EnableDynamicInstall = true };
+            var environment = new TestEnvironment();
+            var installerScriptSnippet = "##INSTALLER_SCRIPT##";
+            var versionProvider = new TestPythonVersionProvider(new[] { "3.7.5", "3.8.0" }, defaultVersion: "3.7.5");
+            var platformInstaller = new TestPythonPlatformInstaller(
+                isVersionAlreadyInstalled: true,
+                installerScript: installerScriptSnippet,
+                Options.Create(options),
+                environment);
+            var platform = CreatePlatform(environment, versionProvider, platformInstaller, options);
+            var repo = new MemorySourceRepo();
+            repo.AddFile("", PythonConstants.RequirementsFileName);
+            repo.AddFile("print(1)", "bla.py");
+            var context = new BuildScriptGeneratorContext { SourceRepo = repo, PythonVersion = "3.7.5" };
+
+            // Act
+            var snippet = platform.GenerateBashBuildScriptSnippet(context);
+
+            // Assert
+            Assert.NotNull(snippet);
+            Assert.Null(snippet.PlatformInstallationScriptSnippet);
+            Assert.Contains(ManifestFilePropertyKeys.PythonVersion, snippet.BuildProperties.Keys);
+            Assert.Equal("3.7.5", snippet.BuildProperties[ManifestFilePropertyKeys.PythonVersion]);
+        }
+
         [Fact]
         public void GeneratedScript_DoesNotUseVenv()
         {
             // Arrange
-            var scriptGenerator = CreatePlatformInstance();
+            var scriptGenerator = CreatePlatform();
             var repo = new MemorySourceRepo();
             repo.AddFile("", PythonConstants.RequirementsFileName);
             repo.AddFile("print(1)", "bla.py");
@@ -42,7 +130,7 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             string compressedVirtualEnvFileName)
         {
             // Arrange
-            var scriptGenerator = CreatePlatformInstance();
+            var scriptGenerator = CreatePlatform();
             var repo = new MemorySourceRepo();
             repo.AddFile("", PythonConstants.RequirementsFileName);
             var venvName = "bla";
@@ -64,23 +152,82 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
             Assert.DoesNotContain(compressedVirtualEnvFileName, excludedDirs);
         }
 
-        private PythonPlatform CreatePlatformInstance(string defaultVersion = null)
+        private PythonPlatform CreatePlatform(
+            IEnvironment environment,
+            IPythonVersionProvider pythonVersionProvider,
+            PythonPlatformInstaller platformInstaller,
+            BuildScriptGeneratorOptions options)
         {
-            var testEnv = new TestEnvironment();
-            testEnv.Variables[PythonConstants.PythonDefaultVersionEnvVarName] = defaultVersion;
-
-            var nodeVersionProvider = new TestVersionProvider(new[] { Common.PythonVersions.Python37Version });
-
-            var scriptGeneratorOptions = Options.Create(new PythonScriptGeneratorOptions());
-            var optionsSetup = new PythonScriptGeneratorOptionsSetup(testEnv);
-            optionsSetup.Configure(scriptGeneratorOptions.Value);
+            var commonOptions = Options.Create(options);
 
             return new PythonPlatform(
-                scriptGeneratorOptions,
-                nodeVersionProvider,
+                commonOptions,
+                pythonVersionProvider,
+                environment,
+                NullLogger<PythonPlatform>.Instance,
+                detector: null,
+                platformInstaller);
+        }
+
+        private PythonPlatform CreatePlatform(string defaultVersion = null)
+        {
+            var testEnv = new TestEnvironment();
+            var versionProvider = new TestPythonVersionProvider(
+                supportedVersions: new[] { Common.PythonVersions.Python37Version },
+                defaultVersion: defaultVersion);
+            var commonOptions = Options.Create(new BuildScriptGeneratorOptions());
+
+            return new PythonPlatform(
+                commonOptions,
+                versionProvider,
                 testEnv,
                 NullLogger<PythonPlatform>.Instance,
-                detector: null);
+                detector: null,
+                new PythonPlatformInstaller(commonOptions, testEnv));
+        }
+
+        private class TestPythonVersionProvider : IPythonVersionProvider
+        {
+            private readonly IEnumerable<string> _supportedVersions;
+            private readonly string _defaultVersion;
+
+            public TestPythonVersionProvider(IEnumerable<string> supportedVersions, string defaultVersion)
+            {
+                _supportedVersions = supportedVersions;
+                _defaultVersion = defaultVersion;
+            }
+
+            public PlatformVersionInfo GetVersionInfo()
+            {
+                return PlatformVersionInfo.CreateOnDiskVersionInfo(_supportedVersions, _defaultVersion);
+            }
+        }
+
+        private class TestPythonPlatformInstaller : PythonPlatformInstaller
+        {
+            private readonly bool _isVersionAlreadyInstalled;
+            private readonly string _installerScript;
+
+            public TestPythonPlatformInstaller(
+                bool isVersionAlreadyInstalled,
+                string installerScript,
+                IOptions<BuildScriptGeneratorOptions> commonOptions,
+                IEnvironment environment)
+                : base(commonOptions, environment)
+            {
+                _isVersionAlreadyInstalled = isVersionAlreadyInstalled;
+                _installerScript = installerScript;
+            }
+
+            public override bool IsVersionAlreadyInstalled(string version)
+            {
+                return _isVersionAlreadyInstalled;
+            }
+
+            public override string GetInstallerScriptSnippet(string version)
+            {
+                return _installerScript;
+            }
         }
     }
 }

tests/BuildScriptGenerator.Tests/Python/PythonVersionProviderTest.cs

@@ -0,0 +1,115 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System.Net.Http;
+using Microsoft.Extensions.Options;
+using Microsoft.Oryx.BuildScriptGenerator.Python;
+using Microsoft.Oryx.Tests.Common;
+using Xunit;
+
+namespace Microsoft.Oryx.BuildScriptGenerator.Tests.Python
+{
+    public class PythonVersionProviderTest
+    {
+        [Fact]
+        public void GetsVersions_FromStorage_WhenDynamicInstall_IsEnabled()
+        {
+            // Arrange
+            var (versionProvider, onDiskVersionProvider, storageVersionProvider) = CreateVersionProvider(
+                enableDynamicInstall: true);
+
+            // Act
+            var versionInfo = versionProvider.GetVersionInfo();
+
+            // Assert
+            Assert.True(storageVersionProvider.GetVersionInfoCalled);
+            Assert.False(onDiskVersionProvider.GetVersionInfoCalled);
+        }
+
+        [Fact]
+        public void GetsVersions_DoesNotGetVersionsFromStorage_WhenDynamicInstall_IsFalse()
+        {
+            // Arrange
+            var (versionProvider, onDiskVersionProvider, storageVersionProvider) = CreateVersionProvider(
+                enableDynamicInstall: false);
+
+            // Act
+            var versionInfo = versionProvider.GetVersionInfo();
+
+            // Assert
+            Assert.False(storageVersionProvider.GetVersionInfoCalled);
+            Assert.True(onDiskVersionProvider.GetVersionInfoCalled);
+        }
+
+        [Fact]
+        public void GetsVersions_DoesNotGetVersionsFromStorage_ByDefault()
+        {
+            // Arrange
+            var (versionProvider, onDiskVersionProvider, storageVersionProvider) = CreateVersionProvider(
+                enableDynamicInstall: false);
+
+            // Act
+            var versionInfo = versionProvider.GetVersionInfo();
+
+            // Assert
+            Assert.False(storageVersionProvider.GetVersionInfoCalled);
+            Assert.True(onDiskVersionProvider.GetVersionInfoCalled);
+        }
+
+        private class TestPythonSdkStorageVersionProvider : PythonSdkStorageVersionProvider
+        {
+            public TestPythonSdkStorageVersionProvider(
+                IEnvironment environment, IHttpClientFactory httpClientFactory)
+                : base(environment, httpClientFactory)
+            {
+            }
+
+            public bool GetVersionInfoCalled { get; private set; }
+
+            public override PlatformVersionInfo GetVersionInfo()
+            {
+                GetVersionInfoCalled = true;
+
+                return null;
+            }
+        }
+
+        private (IPythonVersionProvider, TestPythonOnDiskVersionProvider, TestPythonSdkStorageVersionProvider)
+            CreateVersionProvider(bool enableDynamicInstall)
+        {
+            var commonOptions = Options.Create(new BuildScriptGeneratorOptions()
+            {
+                EnableDynamicInstall = enableDynamicInstall
+            });
+            var pythonOptions = Options.Create(new PythonScriptGeneratorOptions());
+            var environment = new TestEnvironment();
+
+            var onDiskProvider = new TestPythonOnDiskVersionProvider(pythonOptions);
+            var storageProvider = new TestPythonSdkStorageVersionProvider(environment, new TestHttpClientFactory());
+            var versionProvider = new PythonVersionProvider(
+                commonOptions,
+                onDiskProvider,
+                storageProvider);
+            return (versionProvider, onDiskProvider, storageProvider);
+        }
+
+
+        private class TestPythonOnDiskVersionProvider : PythonOnDiskVersionProvider
+        {
+            public TestPythonOnDiskVersionProvider(IOptions<PythonScriptGeneratorOptions> options) : base(options)
+            {
+            }
+
+            public bool GetVersionInfoCalled { get; private set; }
+
+            public override PlatformVersionInfo GetVersionInfo()
+            {
+                GetVersionInfoCalled = true;
+
+                return null;
+            }
+        }
+    }
+}

tests/BuildScriptGenerator.Tests/TestVersionProvider.cs

@@ -9,12 +9,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests
 {
     class TestVersionProvider :
         BuildScriptGenerator.Node.INodeVersionProvider,
-        BuildScriptGenerator.Python.IPythonVersionProvider,
         BuildScriptGenerator.DotNetCore.IDotNetCoreVersionProvider
     {
         public TestVersionProvider(string[] supportedVersions, string[] supportedNpmVersions = null)
         {
-            SupportedNodeVersions = SupportedPythonVersions = SupportedDotNetCoreVersions = supportedVersions;
+            SupportedNodeVersions = SupportedDotNetCoreVersions = supportedVersions;
             SupportedNpmVersions = supportedNpmVersions;
         }
 
@@ -22,8 +21,6 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Tests
 
         public IEnumerable<string> SupportedNpmVersions { get; }
 
-        public IEnumerable<string> SupportedPythonVersions { get; }
-
         public IEnumerable<string> SupportedDotNetCoreVersions { get; }
     }
 }

tests/BuildScriptGeneratorCli.Tests/BuildScriptGeneratorCli.Tests.csproj

@@ -10,6 +10,10 @@
     <RootNamespace>Microsoft.Oryx.BuildScriptGeneratorCli.Tests</RootNamespace>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <PlatformTarget>x64</PlatformTarget>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
     <PackageReference Include="xunit" Version="2.4.1" />

tests/BuildScriptGeneratorCli.Tests/BuildScriptGeneratorOptionsHelperTest.cs

@@ -40,6 +40,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Tests
                 shouldPackage: false,
                 requiredOsPackages: null,
                 scriptOnly: false,
+                enableDynamicInstall: false,
                 properties: null);
 
             // Assert
@@ -70,6 +71,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Tests
                 platformVersion: null,
                 shouldPackage: false,
                 requiredOsPackages: null,
+                enableDynamicInstall: false,
                 scriptOnly: false,
                 properties: null);
 
@@ -98,6 +100,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Tests
                 platformVersion: null,
                 shouldPackage: false,
                 requiredOsPackages: null,
+                enableDynamicInstall: false,
                 scriptOnly: false,
                 properties: null);
 
@@ -127,6 +130,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Tests
                 platformVersion: null,
                 shouldPackage: false,
                 requiredOsPackages: null,
+                enableDynamicInstall: false,
                 scriptOnly: false,
                 properties: null);
 
@@ -159,6 +163,7 @@ namespace Microsoft.Oryx.BuildScriptGeneratorCli.Tests
                 shouldPackage: false,
                 requiredOsPackages: null,
                 scriptOnly: false,
+                enableDynamicInstall: false,
                 properties: null);
 
             // Assert

tests/Oryx.BuildImage.Tests/Oryx.BuildImage.Tests.csproj

@@ -10,6 +10,10 @@
     <SignAssembly>true</SignAssembly>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <PlatformTarget>x64</PlatformTarget>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
     <PackageReference Include="xunit" Version="2.4.1" />

tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs

@@ -0,0 +1,66 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System.Collections.Generic;
+using Microsoft.Oryx.BuildScriptGenerator;
+using Microsoft.Oryx.Common;
+using Microsoft.Oryx.Tests.Common;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Oryx.BuildImage.Tests
+{
+    public class PythonDynamicInstallationTest : PythonSampleAppsTestBase
+    {
+        private readonly string DefaultInstallationRootDir = "/opt/python";
+
+        public PythonDynamicInstallationTest(ITestOutputHelper output) : base(output)
+        {
+        }
+
+        [Fact]
+        public void GeneratesScript_AndBuilds()
+        {
+            // Arrange
+            var appName = "flask-app";
+            var volume = CreateSampleAppVolume(appName);
+            var appDir = volume.ContainerDir;
+            var appOutputDir = "/tmp/app-output";
+            var script = new ShellScriptBuilder()
+                .AddCommand(GetSnippetToCleanUpExistingInstallation())
+                .SetEnvironmentVariable(
+                    SdkStorageConstants.SdkStorageBaseUrlKeyName,
+                    SdkStorageConstants.DevSdkStorageBaseUrl)
+                .AddBuildCommand($"{appDir} -o {appOutputDir} --enable-dynamic-install")
+                .ToString();
+
+            // Act
+            var result = _dockerCli.Run(new DockerRunArguments
+            {
+                ImageId = _imageHelper.GetTestSlimBuildImage(),
+                EnvironmentVariables = new List<EnvironmentVariable> { CreateAppNameEnvVar(appName) },
+                Volumes = new List<DockerVolume> { volume },
+                CommandToExecuteOnRun = "/bin/bash",
+                CommandArguments = new[] { "-c", script }
+            });
+
+            // Assert
+            RunAsserts(
+                () =>
+                {
+                    Assert.True(result.IsSuccess);
+                    Assert.Contains(
+                        $"Python Version: {Constants.TemporaryInstallationDirectoryRoot}/python/3.8.1/bin/python3",
+                        result.StdOut);
+                },
+                result.GetDebugInfo());
+        }
+
+        private string GetSnippetToCleanUpExistingInstallation()
+        {
+            return $"rm -rf {DefaultInstallationRootDir}; mkdir -p {DefaultInstallationRootDir}";
+        }
+    }
+}

tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsTest.cs

@@ -352,7 +352,7 @@ namespace Microsoft.Oryx.BuildImage.Tests
                         $"Python Version: /opt/python/{PythonVersions.Python36Version}/bin/python3",
                         result.StdOut);
                     Assert.Contains(
-                       $"{PythonConstants.PythonName}_version=\"{PythonVersions.Python36Version}\"",
+                       $"{ManifestFilePropertyKeys.PythonVersion}=\"{PythonVersions.Python36Version}\"",
                        result.StdOut);
                 },
                 result.GetDebugInfo());

tests/Oryx.Integration.Tests/Database/DatabaseTestsBase.cs

@@ -45,10 +45,19 @@ namespace Microsoft.Oryx.Integration.Tests
             var appDir = volume.ContainerDir;
             var entrypointScript = "./run.sh";
             var bindPortFlag = specifyBindPortFlag ? $"-bindPort {containerPort}" : string.Empty;
-            var script = new ShellScriptBuilder()
-                .AddCommand($"cd {appDir}")
-                .AddCommand($"oryx -appPath {appDir} {bindPortFlag}")
-                .AddCommand(entrypointScript)
+            var scriptBuilder = new ShellScriptBuilder()
+                .AddCommand($"cd {appDir}");
+
+            if (string.Equals("python", language, StringComparison.OrdinalIgnoreCase))
+            {
+                scriptBuilder = scriptBuilder.AddCommand($"oryx create-script -appPath {appDir} {bindPortFlag}");
+            }
+            else
+            {
+                scriptBuilder = scriptBuilder.AddCommand($"oryx -appPath {appDir} {bindPortFlag}");
+            }
+
+            var script = scriptBuilder.AddCommand(entrypointScript)
                 .ToString();
 
             var runtimeImageName = _imageHelper.GetTestRuntimeImage(language, languageVersion);

tests/Oryx.Integration.Tests/Database/SqlServerIntegrationTests.cs

@@ -80,7 +80,7 @@ namespace Microsoft.Oryx.Integration.Tests
             var volume = DockerVolume.CreateMirror(hostDir);
             var appDir = volume.ContainerDir;
             var script = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Oryx.Integration.Tests.csproj

@@ -10,6 +10,10 @@
     <SignAssembly>true</SignAssembly>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <PlatformTarget>x64</PlatformTarget>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
     <PackageReference Include="Polly" Version="7.1.0" />

tests/Oryx.Integration.Tests/Python/Python27EndToEndTests.cs

@@ -31,7 +31,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .AddCommand($"oryx build {appDir} --platform python --language-version 2.7")
                 .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -output {startupFile} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -output {startupFile} -bindPort {ContainerPort}")
                 .AddCommand(startupFile)
                 .ToString();
 
@@ -75,7 +75,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 // Mimic the commands ran by app service in their derived image.
                 .AddCommand("pip install gunicorn")
                 .AddCommand("pip install flask")
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/Python37EndToEndTests.cs

@@ -35,7 +35,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir} --platform python --language-version {pythonVersion}")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -67,7 +67,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir} -p virtualenv_name={virtualEnvName}")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -120,7 +120,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .AddFileExistsCheck($"{appOutputDir}/{virtualEnvName}.{expectedCompressFileNameExtension}")
                 .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appOutputDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appOutputDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -174,7 +174,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .ToString();
             var runScript = new ShellScriptBuilder()
                 .AddCommand(
-                $"oryx -appPath {appOutputDir} -manifestDir {manifestDir} -bindPort {ContainerPort}")
+                $"oryx create-script -appPath {appOutputDir} -manifestDir {manifestDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -215,7 +215,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .ToString();
             var runScript = new ShellScriptBuilder()
                 .AddCommand($"cd {appDir}")
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -273,7 +273,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .ToString();
 
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appOutputDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appOutputDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/PythonBackwardCompatibilityTests.cs

@@ -38,7 +38,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .AddFileDoesNotExistCheck($"{appDir}/{FilePaths.BuildManifestFileName}")
                 .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -virtualEnvName {virtualEnvName} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -virtualEnvName {virtualEnvName} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -81,7 +81,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .AddFileDoesNotExistCheck($"{appDir}/{FilePaths.BuildManifestFileName}")
                 .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -virtualEnvName {virtualEnvName} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -virtualEnvName {virtualEnvName} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/PythonDebuggingTests.cs

@@ -37,7 +37,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appVolume.ContainerDir} --platform python --platform-version {pythonVersion} --debug")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appVolume.ContainerDir} -bindPort {ContainerPort}" +
+                .AddCommand($"oryx create-script -appPath {appVolume.ContainerDir} -bindPort {ContainerPort}" +
                             $" -debugAdapter ptvsd {scriptGenDebugPortArg} -debugWait")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();

tests/Oryx.Integration.Tests/Python/PythonDjangoAppTests.cs

@@ -32,7 +32,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 .AddBuildCommand($"{appDir} --platform python --language-version 3.6 -p virtualenv_name={virtualEnvName}")
                 .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -81,7 +81,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir} --platform python --language-version 3.6")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -137,7 +137,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 // User would do this through app settings
                 .AddCommand("export ENABLE_MULTIPLATFORM_BUILD=true")
                 .AddCommand("export DJANGO_SETTINGS_MODULE=\"reactdjango.settings.local_base\"")
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/PythonDynamicInstallationTest.cs

@@ -0,0 +1,114 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System.Threading.Tasks;
+using Microsoft.Oryx.Common;
+using Microsoft.Oryx.Tests.Common;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Oryx.Integration.Tests
+{
+    public class PythonDynamicInstallationTest : PythonEndToEndTestsBase
+    {
+        private readonly string DefaultSdksRootDir = "/opt/python";
+
+        public PythonDynamicInstallationTest(ITestOutputHelper output, TestTempDirTestFixture testTempDirTestFixture)
+            : base(output, testTempDirTestFixture)
+        {
+        }
+
+        [Theory]
+        [InlineData("2.7")]
+        [InlineData("3")]
+        [InlineData("3.6")]
+        [InlineData("3.7")]
+        public async Task CanBuildAndRunPythonApp(string pythonVersion)
+        {
+            // Arrange
+            var appName = "flask-app";
+            var volume = CreateAppVolume(appName);
+            var appDir = volume.ContainerDir;
+            var buildScript = new ShellScriptBuilder()
+               .AddCommand(GetSnippetToCleanUpExistingInstallation())
+               .SetEnvironmentVariable(
+                    SdkStorageConstants.SdkStorageBaseUrlKeyName,
+                    SdkStorageConstants.DevSdkStorageBaseUrl)
+               .AddCommand(
+                $"oryx build {appDir} --platform python --platform-version {pythonVersion} --enable-dynamic-install")
+               .ToString();
+            var runScript = new ShellScriptBuilder()
+                .SetEnvironmentVariable(
+                    SdkStorageConstants.SdkStorageBaseUrlKeyName,
+                    SdkStorageConstants.DevSdkStorageBaseUrl)
+                .AddCommand($"oryx setupEnv -appPath {appDir}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand(DefaultStartupFilePath)
+                .ToString();
+
+            await EndToEndTestHelper.BuildRunAndAssertAppAsync(
+                appName,
+                _output,
+                new[] { volume },
+                _imageHelper.GetTestSlimBuildImage(),
+                "/bin/bash", new[] { "-c", buildScript },
+                _imageHelper.GetTestRuntimeImage("python", "dynamic"),
+                ContainerPort,
+                "/bin/bash",
+                new[] { "-c", runScript },
+                async (hostPort) =>
+                {
+                    var data = await _httpClient.GetStringAsync($"http://localhost:{hostPort}/");
+                    Assert.Contains("Hello World!", data);
+                });
+        }
+
+        [Fact]
+        public async Task CanBuildAndRunPythonApp_UsingScriptCommandAndSetEnvSwitch()
+        {
+            // Arrange
+            var pythonVersion = "3.7";
+            var appName = "flask-app";
+            var volume = CreateAppVolume(appName);
+            var appDir = volume.ContainerDir;
+            var buildScript = new ShellScriptBuilder()
+               .AddCommand(GetSnippetToCleanUpExistingInstallation())
+               .SetEnvironmentVariable(
+                    SdkStorageConstants.SdkStorageBaseUrlKeyName,
+                    SdkStorageConstants.DevSdkStorageBaseUrl)
+               .AddCommand(
+                $"oryx build {appDir} --platform python --language-version {pythonVersion} --enable-dynamic-install")
+               .ToString();
+            var runScript = new ShellScriptBuilder()
+                .SetEnvironmentVariable(
+                    SdkStorageConstants.SdkStorageBaseUrlKeyName,
+                    SdkStorageConstants.DevSdkStorageBaseUrl)
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand(DefaultStartupFilePath)
+                .ToString();
+
+            await EndToEndTestHelper.BuildRunAndAssertAppAsync(
+                appName,
+                _output,
+                new[] { volume },
+                _imageHelper.GetTestSlimBuildImage(),
+                "/bin/bash", new[] { "-c", buildScript },
+                _imageHelper.GetTestRuntimeImage("python", "dynamic"),
+                ContainerPort,
+                "/bin/bash",
+                new[] { "-c", runScript },
+                async (hostPort) =>
+                {
+                    var data = await _httpClient.GetStringAsync($"http://localhost:{hostPort}/");
+                    Assert.Contains("Hello World!", data);
+                });
+        }
+
+        private string GetSnippetToCleanUpExistingInstallation()
+        {
+            return $"rm -rf {DefaultSdksRootDir}; mkdir -p {DefaultSdksRootDir}";
+        }
+    }
+}

tests/Oryx.Integration.Tests/Python/PythonEndToEndTests.cs

@@ -30,7 +30,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir} --platform python --language-version 3.6")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -70,7 +70,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir}")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -120,7 +120,7 @@ namespace Microsoft.Oryx.Integration.Tests
 
             var runScript = new ShellScriptBuilder()
                 .AddDirectoryDoesNotExistCheck("__oryx_packages__")
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort} -virtualEnvName={virtualEnvName}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort} -virtualEnvName={virtualEnvName}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/PythonPortEnvironmentVariableTests.cs

@@ -31,7 +31,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .ToString();
             var runScript = new ShellScriptBuilder()
                 .AddCommand($"export PORT={ContainerPort}")
-                .AddCommand($"oryx -appPath {appDir}")
+                .AddCommand($"oryx create-script -appPath {appDir}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 
@@ -72,7 +72,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .ToString();
             var runScript = new ShellScriptBuilder()
                 .AddCommand($"export PORT=9095")
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
 

tests/Oryx.Integration.Tests/Python/PythonShapelyAppTests.cs

@@ -31,7 +31,7 @@ namespace Microsoft.Oryx.Integration.Tests
                .AddCommand($"oryx build {appDir} --platform python --language-version {pythonVersion}")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
             var imageVersion = _imageHelper.GetTestRuntimeImage("python", pythonVersion);
@@ -75,7 +75,7 @@ namespace Microsoft.Oryx.Integration.Tests
                 $"oryx build {appDir} --platform python --platform-version {pythonVersion} -p packagedir={packageDir}")
                .ToString();
             var runScript = new ShellScriptBuilder()
-                .AddCommand($"oryx -appPath {appDir} -bindPort {ContainerPort}")
+                .AddCommand($"oryx create-script -appPath {appDir} -bindPort {ContainerPort}")
                 .AddCommand(DefaultStartupFilePath)
                 .ToString();
             var imageVersion = _imageHelper.GetTestRuntimeImage("python", pythonVersion);

tests/Oryx.RuntimeImage.Tests/Oryx.RuntimeImage.Tests.csproj

@@ -10,6 +10,10 @@
     <SignAssembly>true</SignAssembly>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <PlatformTarget>x64</PlatformTarget>
+  </PropertyGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" />
     <PackageReference Include="xunit" Version="2.4.1" />

tests/Oryx.RuntimeImage.Tests/Python/PythonImagesTests.cs

@@ -39,7 +39,7 @@ namespace Microsoft.Oryx.RuntimeImage.Tests
             {
                 ImageId = _imageHelper.GetTestRuntimeImage("python", version),
                 CommandToExecuteOnRun = "oryx",
-                CommandArguments = new[] { " " }
+                CommandArguments = new[] { "version" }
             });
 
             // Assert
@@ -117,7 +117,7 @@ namespace Microsoft.Oryx.RuntimeImage.Tests
             var script = new ShellScriptBuilder()
                 .CreateDirectory(appPath)
                 .CreateFile(appPath + "/entry.sh", $"exit {exitCodeSentinel}")
-                .AddCommand("oryx -userStartupCommand entry.sh -appPath " + appPath)
+                .AddCommand("oryx create-script -userStartupCommand entry.sh -appPath " + appPath)
                 .AddCommand(". ./run.sh") // Source the default output path
                 .ToString();
 

tests/Oryx.Tests.Common/TestHttpClientFactory.cs

@@ -0,0 +1,17 @@
+// --------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+// --------------------------------------------------------------------------------------------
+
+using System.Net.Http;
+
+namespace Microsoft.Oryx.Tests.Common
+{
+    public class TestHttpClientFactory : IHttpClientFactory
+    {
+        public HttpClient CreateClient(string name)
+        {
+            return new HttpClient();
+        }
+    }
+}