diff --git a/build/__pythonVersions.sh b/build/__pythonVersions.sh index cabeacdc6..d31b82525 100644 --- a/build/__pythonVersions.sh +++ b/build/__pythonVersions.sh @@ -1,8 +1,8 @@ # This file was auto-generated from 'constants.yaml'. Changes may be overridden. PYTHON_BASE_TAG='20200217.4' -PIP_VERSION='19.3.1' +PIP_VERSION='20.0.2' PYTHON27_VERSION='2.7.17' PYTHON36_VERSION='3.6.10' PYTHON37_VERSION='3.7.6' -PYTHON38_VERSION='3.8.1' +PYTHON38_VERSION='3.8.2' diff --git a/build/constants.yaml b/build/constants.yaml index 78e93232e..1f73cdc1b 100644 --- a/build/constants.yaml +++ b/build/constants.yaml @@ -47,11 +47,11 @@ - name: python-versions constants: python-base-tag: 20200217.4 - pip-version: 19.3.1 + pip-version: 20.0.2 python27-version: 2.7.17 python36-version: 3.6.10 python37-version: 3.7.6 - python38-version: 3.8.1 + python38-version: 3.8.2 outputs: - type: csharp directory: src/BuildScriptGenerator diff --git a/images/build/python/2.7/Dockerfile b/images/build/python/2.7/Dockerfile deleted file mode 100644 index 3c88d7dcc..000000000 --- a/images/build/python/2.7/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# Build Python 2.7 -FROM python-build-prereqs -WORKDIR /usr/src/python -ENV GPG_KEY=C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF -RUN . /tmp/__pythonVersions.sh && PYTHON_VERSION=$PYTHON27_VERSION PIP_VERSION=$PIP_VERSION /tmp/build.sh diff --git a/images/build/python/3.6/Dockerfile b/images/build/python/3.6/Dockerfile deleted file mode 100644 index e1b956d0d..000000000 --- a/images/build/python/3.6/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# Build Python 3.6 -FROM python-build-prereqs -WORKDIR /usr/src/python -ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D -RUN . /tmp/__pythonVersions.sh && PYTHON_VERSION=$PYTHON36_VERSION PIP_VERSION=$PIP_VERSION /tmp/build.sh diff --git a/images/build/python/3.7/Dockerfile b/images/build/python/3.7/Dockerfile deleted file mode 100644 index b679d0797..000000000 --- a/images/build/python/3.7/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# Build Python 3.7 -FROM python-build-prereqs -WORKDIR /usr/src/python -ENV GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D -RUN . /tmp/__pythonVersions.sh && PYTHON_VERSION=$PYTHON37_VERSION PIP_VERSION=$PIP_VERSION /tmp/build.sh diff --git a/images/build/python/3.8/Dockerfile b/images/build/python/3.8/Dockerfile deleted file mode 100644 index 07ee9a5bc..000000000 --- a/images/build/python/3.8/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -# Build Python 3.8 -FROM python-build-prereqs -WORKDIR /usr/src/python -ENV GPG_KEY=E3FF2839C048B25C084DEBE9B26995E310250568 -RUN . /tmp/__pythonVersions.sh && PYTHON_VERSION=$PYTHON38_VERSION PIP_VERSION=$PIP_VERSION /tmp/build.sh diff --git a/images/build/python/prereqs/Dockerfile b/images/build/python/prereqs/Dockerfile deleted file mode 100644 index be3bf9389..000000000 --- a/images/build/python/prereqs/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -# Install Python build prerequisites -FROM buildpack-deps:stretch -COPY build/__pythonVersions.sh /tmp/ - -COPY images/build/python/prereqs/build.sh /tmp/ -COPY images/receiveGpgKeys.sh /tmp/receiveGpgKeys.sh -RUN chmod +x /tmp/receiveGpgKeys.sh -RUN chmod +x /tmp/build.sh && \ - apt-get update \ - && apt-get upgrade -y \ - && apt-get install -y --no-install-recommends \ - tk-dev \ - uuid-dev \ - libgeos-dev diff --git a/images/build/python/prereqs/build.sh b/images/build/python/prereqs/build.sh deleted file mode 100755 index 8dbda1bfd..000000000 --- a/images/build/python/prereqs/build.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT license. -# -------------------------------------------------------------------------------------------- - -set -ex - -wget https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz -O /python.tar.xz -wget https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc -O /python.tar.xz.asc - -# Try getting the keys 5 times at most -/tmp/receiveGpgKeys.sh $GPG_KEY - -gpg --batch --verify /python.tar.xz.asc /python.tar.xz -tar -xJf /python.tar.xz --strip-components=1 -C . - -INSTALLATION_PREFIX=/opt/python/$PYTHON_VERSION - -if [ "${PYTHON_VERSION::1}" == "2" ]; then - ./configure \ - --prefix=$INSTALLATION_PREFIX \ - --build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE) \ - --enable-shared \ - --enable-unicode=ucs4 -else - ./configure \ - --prefix=$INSTALLATION_PREFIX \ - --build=$(dpkg-architecture --query DEB_BUILD_GNU_TYPE) \ - --enable-loadable-sqlite-extensions \ - --enable-shared \ - --with-system-expat \ - --with-system-ffi \ - --without-ensurepip -fi - -make -j $(nproc) - -make install - -# Install pip -wget https://bootstrap.pypa.io/get-pip.py -O /get-pip.py -LD_LIBRARY_PATH=/usr/src/python \ -/usr/src/python/python /get-pip.py \ - --prefix $INSTALLATION_PREFIX \ - --disable-pip-version-check \ - --no-cache-dir \ - --no-warn-script-location \ - pip==$PIP_VERSION - -if [ "${PYTHON_VERSION::1}" == "2" ]; then - LD_LIBRARY_PATH=$INSTALLATION_PREFIX/lib \ - $INSTALLATION_PREFIX/bin/pip install --no-cache-dir virtualenv -fi - -# Currently only for version '2' of Python, the alias 'python' exists in the 'bin' -# directory. So to make sure other versions also have this alias, we create the link -# explicitly here. This is for the scenarios where a user does 'benv python=3.7' and -# expects the alias 'python' to point to '3.7' rather than '2'. In cases where benv is -# not passed as an explicit python version, the version '2' is used by default. This is -# done in the Dockerfile. -pythonBinDir="$INSTALLATION_PREFIX/bin" -pythonAliasFile="$pythonBinDir/python" -if [ ! -e "$pythonAliasFile" ]; then - IFS='.' read -ra SPLIT_VERSION <<< "$PYTHON_VERSION" - majorAndMinorParts="${SPLIT_VERSION[0]}.${SPLIT_VERSION[1]}" - ln -s $pythonBinDir/python$majorAndMinorParts $pythonBinDir/python -fi - -# Replace log level in pip's code as a workaround for https://github.com/pypa/pip/issues/6189 -pipReqSetPath=`find $INSTALLATION_PREFIX/lib -path "*site-packages/pip/_internal/req/req_set.py"` -sed -i 's|logger\.debug('\''Cleaning up\.\.\.'\'')|logger\.info('\''Cleaning up\.\.\.'\'')|' "$pipReqSetPath" diff --git a/images/installPlatform.sh b/images/installPlatform.sh index d7d3a0cf0..3061e22e3 100644 --- a/images/installPlatform.sh +++ b/images/installPlatform.sh @@ -48,11 +48,18 @@ if [ -z "$targetDir" ]; then targetDir="$platformDir/$VERSION" fi +START_TIME=$SECONDS downloadFileAndVerifyChecksum $PLATFORM_NAME $VERSION $fileName +ELAPSED_TIME=$(($SECONDS - $START_TIME)) +echo "Downloaded and verified checksum in $ELAPSED_TIME sec(s)." +echo "Extracting..." +START_TIME=$SECONDS mkdir -p "$targetDir" tar -xzf $fileName -C "$targetDir" rm -f "$fileName" +ELAPSED_TIME=$(($SECONDS - $START_TIME)) +echo "Extracted contents in $ELAPSED_TIME sec(s)." if [ "$createLinks" != "false" ]; then # Create a link : major.minor => major.minor.patch diff --git a/images/runtime/dotnetcore/2.2/base.Dockerfile b/images/runtime/dotnetcore/2.2/base.Dockerfile index 9a93ebd35..78300e844 100644 --- a/images/runtime/dotnetcore/2.2/base.Dockerfile +++ b/images/runtime/dotnetcore/2.2/base.Dockerfile @@ -36,7 +36,7 @@ ENV PATH="/opt/dotnetcore-tools:${PATH}" ENV ASPNETCORE_VERSION 2.2.7 RUN curl -SL --output aspnetcore.tar.gz https://dotnetcli.blob.core.windows.net/dotnet/aspnetcore/Runtime/$ASPNETCORE_VERSION/aspnetcore-runtime-$ASPNETCORE_VERSION-linux-x64.tar.gz \ - && aspnetcore_sha512=''3fdc874a20d5cd318deabf12d73d26bd1f9b767cf351d05bfed5efc6d66c1d774ebd911d7dc28a5a7f6af9976d50068b217ef051024d3c91496d4a44b89b374a' \ + && aspnetcore_sha512='3fdc874a20d5cd318deabf12d73d26bd1f9b767cf351d05bfed5efc6d66c1d774ebd911d7dc28a5a7f6af9976d50068b217ef051024d3c91496d4a44b89b374a' \ && echo "$aspnetcore_sha512 aspnetcore.tar.gz" | sha512sum -c - \ && mkdir -p /usr/share/dotnet \ && tar -zxf aspnetcore.tar.gz -C /usr/share/dotnet \ diff --git a/platforms/python/buildPython.sh b/platforms/python/buildPython.sh index 25b5192bc..578c709eb 100755 --- a/platforms/python/buildPython.sh +++ b/platforms/python/buildPython.sh @@ -28,8 +28,7 @@ buildPythonPrereqsImage() { buildPython() { local version="$1" local gpgKey="$2" - local pipVersion="$3" - local dockerFile="$4" + local dockerFile="$3" local imageName="oryx/python" if shouldBuildSdk python python-$version.tar.gz; then @@ -46,17 +45,12 @@ buildPython() { else dockerFile="$pythonPlatformDir/$dockerFile" fi - - if [ -z "$pipVersion" ]; then - # Use default pip version - pipVersion="$PIP_VERSION" - fi docker build \ -f "$dockerFile" \ --build-arg VERSION_TO_BUILD=$version \ --build-arg GPG_KEYS=$gpgKey \ - --build-arg PIP_VERSION=$pipVersion \ + --build-arg PIP_VERSION=$PIP_VERSION \ -t $imageName \ $REPO_DIR diff --git a/platforms/python/versionsToBuild.txt b/platforms/python/versionsToBuild.txt index 200fe51b5..0b3a946ca 100644 --- a/platforms/python/versionsToBuild.txt +++ b/platforms/python/versionsToBuild.txt @@ -1,4 +1,4 @@ -# version, gpg keys, pipVersion +# version, gpg keys, dockerfile 2.7.16, C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF, 2.7.17, C01E1CAD5EA2C4F0B8E3571504C367C218ADD4FF, 3.6.9, 0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D, @@ -11,6 +11,7 @@ 3.8.0b3, E3FF2839C048B25C084DEBE9B26995E310250568, 3.8.0b4, E3FF2839C048B25C084DEBE9B26995E310250568, 3.8.1, E3FF2839C048B25C084DEBE9B26995E310250568, +3.8.2, E3FF2839C048B25C084DEBE9B26995E310250568, # NOTE: Make sure to set the default version in 'defaultVersion.txt' file diff --git a/src/BuildScriptGenerator/PythonVersions.cs b/src/BuildScriptGenerator/PythonVersions.cs index 33f94e412..5491249b9 100644 --- a/src/BuildScriptGenerator/PythonVersions.cs +++ b/src/BuildScriptGenerator/PythonVersions.cs @@ -5,10 +5,10 @@ namespace Microsoft.Oryx.Common public static class PythonVersions { public const string PythonBaseTag = "20200217.4"; - public const string PipVersion = "19.3.1"; + public const string PipVersion = "20.0.2"; public const string Python27Version = "2.7.17"; public const string Python36Version = "3.6.10"; public const string Python37Version = "3.7.6"; - public const string Python38Version = "3.8.1"; + public const string Python38Version = "3.8.2"; } } \ No newline at end of file diff --git a/tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs b/tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs index 53ef6498e..6d7c7863b 100644 --- a/tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs +++ b/tests/Oryx.BuildImage.Tests/Python/PythonDynamicInstallationTest.cs @@ -32,46 +32,46 @@ namespace Microsoft.Oryx.BuildImage.Tests } } - //[Theory] - //[MemberData(nameof(ImageNameData))] - //public void GeneratesScript_AndBuilds(string imageName) - //{ - // // Arrange - // var version = "3.8.1"; - // var appName = "flask-app"; - // var volume = CreateSampleAppVolume(appName); - // var appDir = volume.ContainerDir; - // var appOutputDir = "/tmp/app-output"; - // var script = new ShellScriptBuilder() - // .AddCommand(GetSnippetToCleanUpExistingInstallation()) - // .SetEnvironmentVariable( - // SdkStorageConstants.SdkStorageBaseUrlKeyName, - // SdkStorageConstants.DevSdkStorageBaseUrl) - // .AddBuildCommand( - // $"{appDir} --platform python --platform-version {version} -o {appOutputDir} --enable-dynamic-install") - // .ToString(); + [Theory] + [MemberData(nameof(ImageNameData))] + public void GeneratesScript_AndBuilds(string imageName) + { + // Arrange + var version = "3.8.1"; + var appName = "flask-app"; + var volume = CreateSampleAppVolume(appName); + var appDir = volume.ContainerDir; + var appOutputDir = "/tmp/app-output"; + var script = new ShellScriptBuilder() + .AddCommand(GetSnippetToCleanUpExistingInstallation()) + .SetEnvironmentVariable( + SdkStorageConstants.SdkStorageBaseUrlKeyName, + SdkStorageConstants.DevSdkStorageBaseUrl) + .AddBuildCommand( + $"{appDir} --platform python --platform-version {version} -o {appOutputDir} --enable-dynamic-install") + .ToString(); - // // Act - // var result = _dockerCli.Run(new DockerRunArguments - // { - // ImageId = imageName, - // EnvironmentVariables = new List { CreateAppNameEnvVar(appName) }, - // Volumes = new List { volume }, - // CommandToExecuteOnRun = "/bin/bash", - // CommandArguments = new[] { "-c", script } - // }); + // Act + var result = _dockerCli.Run(new DockerRunArguments + { + ImageId = imageName, + EnvironmentVariables = new List { CreateAppNameEnvVar(appName) }, + Volumes = new List { volume }, + CommandToExecuteOnRun = "/bin/bash", + CommandArguments = new[] { "-c", script } + }); - // // Assert - // RunAsserts( - // () => - // { - // Assert.True(result.IsSuccess); - // Assert.Contains( - // $"Python Version: {Constants.TemporaryInstallationDirectoryRoot}/python/{version}/bin/python3", - // result.StdOut); - // }, - // result.GetDebugInfo()); - //} + // Assert + RunAsserts( + () => + { + Assert.True(result.IsSuccess); + Assert.Contains( + $"Python Version: {Constants.TemporaryInstallationDirectoryRoot}/python/{version}/bin/python3", + result.StdOut); + }, + result.GetDebugInfo()); + } private string GetSnippetToCleanUpExistingInstallation() { diff --git a/vsts/scripts/publishFilesToAzureStorage.sh b/vsts/scripts/publishFilesToAzureStorage.sh index 2db96e5bc..df45fe579 100755 --- a/vsts/scripts/publishFilesToAzureStorage.sh +++ b/vsts/scripts/publishFilesToAzureStorage.sh @@ -16,7 +16,7 @@ uploadFiles() { return fi - allFiles=$(find $artifactsDir -type f -name '*.tar.gz' -o -name '*.txt') + allFiles=$(find $artifactsDir -type f -name '*.tar.gz' -o -name 'defaultVersion.txt') for fileToUpload in $allFiles do fileName=$(basename $fileToUpload)