add some missing python modules and update python platform versions with latest patches (#1117)

* add missing python bz2 module * update python platform versions * update * update default python versions * add ldconfig update for python dynamic installation * fix tests * update script, remove debug commands
2021-10-04 15:23:16 -07:00 · 2021-10-04 15:23:16 -07:00 · 135f631893
--- a/build/__pythonVersions.sh
+++ b/build/__pythonVersions.sh
@ -2,7 +2,7 @@

 PIP_VERSION='20.2.3'
 PYTHON27_VERSION='2.7.18'
-PYTHON36_VERSION='3.6.12'
-PYTHON37_VERSION='3.7.9'
-PYTHON38_VERSION='3.8.6'
-PYTHON39_VERSION='3.9.0'
+PYTHON36_VERSION='3.6.15'
+PYTHON37_VERSION='3.7.12'
+PYTHON38_VERSION='3.8.12'
+PYTHON39_VERSION='3.9.7'
--- a/build/constants.yaml
+++ b/build/constants.yaml
@ -97,10 +97,10 @@
  constants:
    pip-version: 20.2.3
    python27-version: 2.7.18
-    python36-version: 3.6.12
-    python37-version: 3.7.9
-    python38-version: 3.8.6
-    python39-version: 3.9.0
+    python36-version: 3.6.15
+    python37-version: 3.7.12
+    python38-version: 3.8.12
+    python39-version: 3.9.7
  outputs:
    - type: csharp
      directory: src/BuildScriptGenerator
--- a/images/installPlatform.sh
+++ b/images/installPlatform.sh
@ -72,6 +72,12 @@ rm -f "$fileName"
 ELAPSED_TIME=$(($SECONDS - $START_TIME))
 echo "Extracted contents in $ELAPSED_TIME sec(s)."

+if [ "$PLATFORM_NAME" == "python" ]
+then
+   [ -d "/opt/python/$VERSION" ] && echo /opt/python/$VERSION/lib >> /etc/ld.so.conf.d/python.conf
+   ldconfig
+fi
+
 if [ "$createLinks" != "false" ]; then
    # Create a link : major.minor => major.minor.patch
    cd "$platformDir"
--- a/platforms/python/prereqs/build.sh
+++ b/platforms/python/prereqs/build.sh
@ -20,12 +20,13 @@ else
    # for buster and ubuntu we would need following libraries to build php 
    apt-get update && \
 	apt-get upgrade -y && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-		libssl-dev \
-		libncurses5-dev \
-		libsqlite3-dev \
-		libreadline-dev \
-		libgdm-dev
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        libssl-dev \
+        libncurses5-dev \
+        libsqlite3-dev \
+        libreadline-dev \
+        libbz2-dev \
+        libgdm-dev
 fi

 # Try getting the keys 5 times at most
--- a/platforms/python/versionsToBuild.txt
+++ b/platforms/python/versionsToBuild.txt
@ -1,14 +1,17 @@
 # version, gpg keys, dockerfile
 3.6.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,
+3.6.15, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

 3.7.9, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,
+3.7.12, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D,

 3.8.6, E3FF2839C048B25C084DEBE9B26995E310250568,
+3.8.12, E3FF2839C048B25C084DEBE9B26995E310250568,

 3.9.0, E3FF2839C048B25C084DEBE9B26995E310250568,
 3.9.1rc1, E3FF2839C048B25C084DEBE9B26995E310250568,
+3.9.7, E3FF2839C048B25C084DEBE9B26995E310250568,

 3.10.0a2, 64E628F8D684696D,

-
 # NOTE: Make sure to set the default version in 'defaultVersion.txt' file
--- a/src/BuildScriptGenerator/PlatformInstallerBase.cs
+++ b/src/BuildScriptGenerator/PlatformInstallerBase.cs
@ -79,13 +79,11 @@ namespace Microsoft.Oryx.BuildScriptGenerator
                .AppendLine($"echo \"performing sha512 checksum for: {platformName}...\"")
                .AppendLine($"echo \"$checksumValue {version}.tar.gz\" | sha512sum -c - >/dev/null 2>&1")
                .AppendLine("fi")
-
                .AppendLine($"rm -f {tarFile}")
                .AppendLine("PLATFORM_SETUP_ELAPSED_TIME=$(($SECONDS - $PLATFORM_SETUP_START))")
                .AppendLine("echo \"Done in $PLATFORM_SETUP_ELAPSED_TIME sec(s).\"")
                .AppendLine("echo")
                .AppendLine("oryxImageDetectorFile=\"/opt/oryx/.imagetype\"")
-
                .AppendLine($"if [ -f \"$oryxImageDetectorFile\" ] && [ \"$platformName\" = \"dotnet\" ] && grep -q \"jamstack\" \"$oryxImageDetectorFile\"; then")
                .AppendLine("echo \"image detector file exists, platform is dotnet..\"")
                .AppendLine($"export dotnet=/opt/dotnet/{version}/dotnet")
@ -99,31 +97,28 @@ namespace Microsoft.Oryx.BuildScriptGenerator
                .AppendLine("echo \"image detector file exists, platform is nodejs..\"")
                .AppendLine($"mkdir -p /home/codespace/.nodejs")
                .AppendLine($"ln -sfn /opt/nodejs/{version} /home/codespace/.nodejs/current")
-                .AppendLine($"ls -la /home/codespace/.nodejs/current")
                .AppendLine("fi")
                .AppendLine($"if [ -f \"$oryxImageDetectorFile\" ] && [ \"$platformName\" = \"php\" ] && grep -q \"vso-focal\" \"$oryxImageDetectorFile\"; then")
                .AppendLine("echo \"image detector file exists, platform is php..\"")
                .AppendLine($"mkdir -p /home/codespace/.php")
                .AppendLine($"ln -sfn /opt/php/{version} /home/codespace/.php/current")
-                .AppendLine($"ls -la /home/codespace/.php/current")
                .AppendLine("fi")
                .AppendLine($"if [ -f \"$oryxImageDetectorFile\" ] && [ \"$platformName\" = \"python\" ] && grep -q \"vso-focal\" \"$oryxImageDetectorFile\"; then")
-                .AppendLine("echo \"image detector file exists, platform is python..\"")
-                .AppendLine($"mkdir -p /home/codespace/.python")
-                .AppendLine($"ln -sfn /opt/python/{version} /home/codespace/.python/current")
-                .AppendLine($"ls -la /home/codespace/.python/current")
+                .AppendLine("   echo \"image detector file exists, platform is python..\"")
+                .AppendLine($"  [ -d \"/opt/python/$VERSION\" ] && echo /opt/python/{version}/lib >> /etc/ld.so.conf.d/python.conf")
+                .AppendLine($"  ldconfig")
+                .AppendLine($"  mkdir -p /home/codespace/.python")
+                .AppendLine($"  ln -sfn /opt/python/{version} /home/codespace/.python/current")
                .AppendLine("fi")
                .AppendLine($"if [ -f \"$oryxImageDetectorFile\" ] && [ \"$platformName\" = \"java\" ] && grep -q \"vso-focal\" \"$oryxImageDetectorFile\"; then")
                .AppendLine("echo \"image detector file exists, platform is java..\"")
                .AppendLine($"mkdir -p /home/codespace/.java")
                .AppendLine($"ln -sfn /opt/java/{version} /home/codespace/.java/current")
-                .AppendLine($"ls -la /home/codespace/.java/current")
                .AppendLine("fi")
                .AppendLine($"if [ -f \"$oryxImageDetectorFile\" ] && [ \"$platformName\" = \"ruby\" ] && grep -q \"vso-focal\" \"$oryxImageDetectorFile\"; then")
                .AppendLine("echo \"image detector file exists, platform is ruby..\"")
                .AppendLine($"mkdir -p /home/codespace/.ruby")
                .AppendLine($"ln -sfn /opt/ruby/{version} /home/codespace/.ruby/current")
-                .AppendLine($"ls -la /home/codespace/.ruby/current")
                .AppendLine("fi")

                // Write out a sentinel file to indicate downlaod and extraction was successful
--- a/src/BuildScriptGenerator/PythonVersions.cs
+++ b/src/BuildScriptGenerator/PythonVersions.cs
@ -6,9 +6,9 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Common
    {
        public const string PipVersion = "20.2.3";
        public const string Python27Version = "2.7.18";
-        public const string Python36Version = "3.6.12";
-        public const string Python37Version = "3.7.9";
-        public const string Python38Version = "3.8.6";
-        public const string Python39Version = "3.9.0";
+        public const string Python36Version = "3.6.15";
+        public const string Python37Version = "3.7.12";
+        public const string Python38Version = "3.8.12";
+        public const string Python39Version = "3.9.7";
    }
 }
--- a/tests/Oryx.BuildImage.Tests/DotNetCore/DotNetCoreSampleAppsTest.cs
+++ b/tests/Oryx.BuildImage.Tests/DotNetCore/DotNetCoreSampleAppsTest.cs
@ -29,7 +29,7 @@ namespace Microsoft.Oryx.BuildImage.Tests

        private readonly string SdkVersionMessageFormat = "Using .NET Core SDK Version: {0}";

-        [Fact]
+        [Fact (Skip="NetCore11 is no longer officially supported")]
        public void Builds_NetCore10App_UsingNetCore11_DotNetSdkVersion()
        {
            // Arrange
--- a/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsDjangoAppRunningCollectStatic.cs
+++ b/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsDjangoAppRunningCollectStatic.cs
@ -37,7 +37,7 @@ namespace Microsoft.Oryx.BuildImage.Tests
            }
            var script = scriptBuilder
                .AddBuildCommand(
-                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName} --platform-version {PythonVersions.Python37Version}")
+                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName}")
                // These css files should be available since 'collectstatic' is run in the script
                .AddFileExistsCheck($"{appOutputDir}/staticfiles/css/boards.css")
                .AddFileExistsCheck($"{appOutputDir}/staticfiles/css/uservoice.css")
--- a/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsDjangoAppWithoutRunningCollectStaticTest.cs
+++ b/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsDjangoAppWithoutRunningCollectStaticTest.cs
@ -33,7 +33,7 @@ namespace Microsoft.Oryx.BuildImage.Tests
            var script = new ShellScriptBuilder()
                .AddCommand($"export {EnvironmentSettingsKeys.DisableCollectStatic}={disableCollectStatic}")
                .AddBuildCommand(
-                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName} --platform-version {PythonVersions.Python37Version}")
+                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName}")
                // These css files should NOT be available since 'collectstatic' is set off
                .AddFileDoesNotExistCheck($"{appOutputDir}/staticfiles/css/boards.css")
                .AddFileDoesNotExistCheck($"{appOutputDir}/staticfiles/css/uservoice.css")
--- a/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsTest.cs
+++ b/tests/Oryx.BuildImage.Tests/Python/PythonSampleAppsTest.cs
@ -1169,8 +1169,7 @@ namespace Microsoft.Oryx.BuildImage.Tests
            var appOutputDir = "/tmp/app-output";
            var script = new ShellScriptBuilder()
                .AddBuildCommand(
-                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName} " +
-                $"--platform-version {PythonVersions.Python37Version}")
+                $"{appDir} -o {appOutputDir} --platform {PythonConstants.PlatformName}")
                .ToString();

            // Act