From 589b3cfc8222d68a11fdca106f58393e938180f4 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Sun, 23 Oct 2016 15:50:43 +0200 Subject: [PATCH] Scripts/linux/install-cntk.sh: update; include cntk binary installation - Conda environments: install pillow as an Anaconda package. --- .../conda-linux-cntk-py34-environment.yml | 2 +- ...ntkv2_linux_install.sh => install-cntk.sh} | 131 ++++++++++++------ .../conda-windows-cntk-py34-environment.yml | 2 +- Tests/Install/linux/Dockerfile-CPU | 20 +++ Tests/Install/linux/Dockerfile-GPU | 21 +++ Tests/Install/linux/Dockerfile-Ubuntu14-CPU | 20 +++ Tests/Install/linux/Dockerfile-Ubuntu14-GPU | 20 +++ Tests/Install/linux/prep-testenv.sh | 5 + Tests/Install/linux/test.sh | 8 ++ Tests/Install/linux/test_wrapper.sh | 31 +++++ Tests/Install/linux/visudo-helper.sh | 6 + 11 files changed, 219 insertions(+), 47 deletions(-) rename Scripts/linux/{pycntkv2_linux_install.sh => install-cntk.sh} (60%) create mode 100644 Tests/Install/linux/Dockerfile-CPU create mode 100644 Tests/Install/linux/Dockerfile-GPU create mode 100644 Tests/Install/linux/Dockerfile-Ubuntu14-CPU create mode 100644 Tests/Install/linux/Dockerfile-Ubuntu14-GPU create mode 100755 Tests/Install/linux/prep-testenv.sh create mode 100644 Tests/Install/linux/test.sh create mode 100755 Tests/Install/linux/test_wrapper.sh create mode 100755 Tests/Install/linux/visudo-helper.sh diff --git a/Scripts/linux/conda-linux-cntk-py34-environment.yml b/Scripts/linux/conda-linux-cntk-py34-environment.yml index c59d542b2..89c322ddb 100644 --- a/Scripts/linux/conda-linux-cntk-py34-environment.yml +++ b/Scripts/linux/conda-linux-cntk-py34-environment.yml @@ -5,6 +5,7 @@ dependencies: - jupyter=1.0.0=py34_3 - matplotlib=1.5.3=np111py34_0 - numpy=1.11.2=py34_0 +- pillow=3.4.2=py34_0 - pip=8.1.2=py34_0 - python=3.4.4=5 - pyyaml=3.12=py34_0 @@ -13,7 +14,6 @@ dependencies: - six=1.10.0=py34_0 - wheel=0.29.0=py34_0 - pip: - - pillow==3.4.2 - pytest==3.0.3 - sphinx==1.4.8 - sphinx-rtd-theme==0.1.9 diff --git a/Scripts/linux/pycntkv2_linux_install.sh b/Scripts/linux/install-cntk.sh similarity index 60% rename from Scripts/linux/pycntkv2_linux_install.sh rename to Scripts/linux/install-cntk.sh index a091606a9..06f9f7b21 100755 --- a/Scripts/linux/pycntkv2_linux_install.sh +++ b/Scripts/linux/install-cntk.sh @@ -7,15 +7,34 @@ # ============================================================================== # Log steps, stop on error +# TODO cut down on logging set -x -e -o pipefail -USAGE="Usage: $0 [--force]" -CNTK_PIP_URL=${1?$USAGE} -FORCE=$(! [ "$2" = "--force" ]; echo $?) +REPO_TAG=v2.alpha4 -# Change to the script's directory -SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")" -cd "$SCRIPT_DIR" +USAGE="Usage: [--force]" +FORCE=$(! [ "$0" = "--force" ]; echo $?) + +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" + +# Go to the drop root +cd "$SCRIPT_DIR/../.." + +CNTK_BIN_PATH="$PWD/cntk/bin" +CNTK_LIB_PATH="$PWD/cntk/lib" +CNTK_DEP_LIB_PATH="$PWD/cntk/dependencies/lib" +CNTK_EXAMPLES_PATH="$PWD/Examples" +CNTK_BINARY="$CNTK_BIN_PATH/cntk" +CNTK_PY34_ENV_FILE="$SCRIPT_DIR/conda-linux-cntk-py34-environment.yml" +CNTK_WHEEL_PATH="cntk/python/cntk-2.0a4-cp34-cp34m-linux_x86_64.whl" +test -d "$CNTK_BIN_PATH" && test -d "$CNTK_LIB_PATH" && test -d "$CNTK_DEP_LIB_PATH" && +test -d "$CNTK_EXAMPLES_PATH" && test -x "$CNTK_BINARY" && +test -f "$CNTK_PY34_ENV_FILE" && test -f "$CNTK_WHEEL_PATH" || { + echo Cannot find expected drop content. Please double-check that this is a + echo CNTK binary drop for Linux. Go to https://github.com/Microsoft/CNTK/wiki + echo for help. + exit 1 +} # Check for tested OS (note: only a warning, we can live with lsb-release not being available) [[ "$(lsb_release -i)" =~ :.*Ubuntu ]] && [[ "$(lsb_release -r)" =~ :.*(14\.04|16\.04) ]] || { @@ -43,6 +62,9 @@ else PACKAGES+=" openmpi-bin" fi +# TODO for Pillow +#PACKAGES+=" build-essential libjpeg8 libjpeg62-dev libfreetype6 libfreetype6-dev" + if dpkg -s $PACKAGES 1>/dev/null 2>/dev/null; then printf "Packages already installed, skipping.\n" else @@ -50,12 +72,6 @@ else sudo apt-get install -y --no-install-recommends $PACKAGES fi -# Check URL early on for feedback -wget --spider -q "$CNTK_PIP_URL" || { - printf "Please double-check URL '%s', does it exist?\n$USAGE" "$CNTK_PIP_URL" - exit 1 -} - ######################################### # On Ubuntu 14.04: OpenMPI build @@ -99,40 +115,26 @@ fi CONDA="$HOME/anaconda3/bin/conda" [ -x "$CONDA" ] -ACTIVATE="$HOME/anaconda3/bin/activate" -[ -x "$ACTIVATE" ] +PY_ACTIVATE="$HOME/anaconda3/bin/activate" +[ -x "$PY_ACTIVATE" ] CNTK_PY34_ENV_PREFIX="$ANACONDA_PREFIX/envs/cntk-py34" if [ -d "$CNTK_PY34_ENV_PREFIX" ]; then printf "Path '%s' already exists, skipping CNTK Python 3.4 environment setup\n" "$CNTK_PY34_ENV_PREFIX" else - CNTK_PY34_ENV_FILE=conda-linux-cntk-py34-environment.yml - - cat >| "$CNTK_PY34_ENV_FILE" <| "$ACTIVATE_SCRIPT_NAME" < /dev/null)" == "$ACTIVATE_SCRIPT_NAME" ]; then + echo Error: this script is meant to be sourced. Run 'source activate-cntk' +else + export PATH="$CNTK_BIN_PATH:\$PATH" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH_SETTING" + source "$PY_ACTIVATE" "$CNTK_PY34_ENV_PREFIX" + + cat < /home/testuser/test-env.sh +printf "Now do 'su - testuser' then 'source test-env.sh' to set up test env\n" diff --git a/Tests/Install/linux/test.sh b/Tests/Install/linux/test.sh new file mode 100644 index 000000000..7db1810d6 --- /dev/null +++ b/Tests/Install/linux/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e -x + +docker build -t cntkpyv2:cpuonly -f Dockerfile-Ubuntu14-CPU . +docker build -t cntkpyv2:cpuonly-ubuntu16 -f Dockerfile-CPU . + +docker build -t cntkpyv2:gpu -f Dockerfile-Ubuntu14-GPU . +docker build -t cntkpyv2:gpu-ubuntu16 -f Dockerfile-GPU . diff --git a/Tests/Install/linux/test_wrapper.sh b/Tests/Install/linux/test_wrapper.sh new file mode 100755 index 000000000..b7f044a5b --- /dev/null +++ b/Tests/Install/linux/test_wrapper.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# ============================================================================== +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +# Log steps, stop on error +set -x -e -o pipefail + +USAGE="Usage: $0 " + +DROP_LOCATION=${1?$USAGE} + +if [ -f "$DROP_LOCATION" ]; then + DROP_FILE="$DROP_LOCATION" +else + # TODO tune + sudo apt-get install wget ca-certificates + # Not found locally, assume it's a URL + wget "$DROP_LOCATION" + DROP_FILE="$(basename "$DROP_LOCATION")" +fi + +tar -xzf "$DROP_FILE" +test -d cntk + +exec cntk/Scripts/linux/install-cntk.sh + +# vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Tests/Install/linux/visudo-helper.sh b/Tests/Install/linux/visudo-helper.sh new file mode 100755 index 000000000..4f3ebfa2b --- /dev/null +++ b/Tests/Install/linux/visudo-helper.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -x +FILE="$1" +[ "$FILE" = "--" ] && FILE="$2" +printf "testuser\tALL=(ALL) NOPASSWD: ALL\n" >> "$FILE" +# use as: VISUAL=/path/to/script visudo