From f9936fc5e2055aeb92997f27ebeae0edd41ffd58 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Sun, 5 Nov 2017 14:06:52 -0800 Subject: [PATCH] Generate pre-built binaries, Windows docker images - Pre-built CLI binaries for Linux x86_64, Windows amd64 - Docker CLI image for Windows --- .travis.yml | 54 +++++++++++ CHANGELOG.md | 2 + README.md | 3 - appveyor.yml | 89 +++++++++++++++++-- cargo/{windows => }/recurrent_job_manager.cmd | 0 docker/linux/cli/Dockerfile | 2 +- .../windows/cargo}/Dockerfile | 4 +- docker/windows/cli/Dockerfile | 50 +++++++++++ docs/01-batch-shipyard-installation.md | 39 ++++++-- shipyard.py | 3 +- 10 files changed, 225 insertions(+), 21 deletions(-) rename cargo/{windows => }/recurrent_job_manager.cmd (100%) rename {cargo/windows => docker/windows/cargo}/Dockerfile (94%) create mode 100644 docker/windows/cli/Dockerfile diff --git a/.travis.yml b/.travis.yml index 0ece436..93186d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,67 @@ language: python + cache: pip + python: - 2.7 - 3.4 - 3.5 - 3.6 + +env: + global: + - BATCH_SHIPYARD_CLI_ARTIFACT=batch-shipyard-$TRAVIS_TAG-cli-$TRAVIS_OS_NAME-x86_64 + - secure: "CsiojO04pOwYbCduqXcZfSsJZ01n0bc4hflza4tk9ZHLotcZkCRKGVsemdPVKZVNlzy+gUpzyc4Tc4/h0391piQ51CCTThde2csDWbXssoO71b4U/jIuKhOqMHyp+5dGznBXOcFye9d5gpdHF0z3AZbo9V7zkuO+3YD/ES1YLWJ5+6RRAR11FRofipfNFYQ7OPgPzrqI0IHHKOYG02JeN7ZmpX+fEitH/J561y9RMIhEtk2udBd51JSTIYao7IENEpHoD41CwwlpOLWKhr7oo9wmyOTdvwyNzLh6Z/CxBUMvmvKn4zsK2d/WzM77EwN4bcBwu2vJmfR+Hi5CD61L9KdhVM+rWUn8+OJUmiNo+ZYVM5w/cHdyC7BYe5hHDAEVmvg0dQIxFXTov5+PjAi0HjA2RNrI2i2ABX1N8wrtPCxlkL8Pnbb93DfTPWmYQVS6LsXrh77KuD7gRxu1s5KAmlfcG//bGuKvpDz6vm7+/cXxnB4sZX+Et9S/F1GV1VYOAr3YwLuE2Tl8MPyoUJa/PdK+md3klL9WteQWXJHLcLbnt34VvstF0jxGh/hqI/3lKtN1U5bsfRbGNBAvYGbGQKpVleF5/0yDLI5/kWI7bIuEF4FHYgIOSvIAzVdC+GXYAeXgrjlOHDFVKwOGL1lhvY3nA/fbENNhHeEN3RQc6N8=" + - secure: "XgD2xFqMvimEDzwRvUwNmA7EAl6Y/ydwgXXq+O4eEE09Xl/UQvmCZ4soTwFqOUnf798Xqn6R2qXHdqpl2l4IXMILa4LteNUzRKd0txOj2lPGftFs1s09Odu729PkoEs9qbM446yjvK+PhbDUvD9GboQ3sWbzYSUPPDiJuPWiJ5KasooGdyfahV542hBo2iOc0Ezq+sziBwBffY407XcX7mTmXjEtQuwSezJGWynHUSD7gIywCXAuzTqdK3ikYYdSJTUn6SnlZZlLuICoOzVULT97VzA+o+puFU/rSnG9+czUUcXILpDzjpl231b858sQ7RdLocPdTiyIVx06FOfW/XfKUyDbRHGhiVZzsJ4xDMxtQk9haLb5NJjwZGFWmyy2tHnj86yNwxwmocbiRymqC2sz/rcIpmwMneKIGJG9MuXRMlGqeMxCcDb3FoaoUp6JIflkx5qsmgsaTEs4m0tcdyivf6ugW1dpBIGek8sf/LISmdakZTNeJxGUyMNYRFdSRWTTZvQ4fj3ZVxLoTCHLszwwHZ1l3AstbiYk86/5Zn8AS+5cWnVEZP3ayPkWqVKMjLxQrHQsLKGgpJBjDXTUp4NixssgXWmDmJMuME0TEjoT58hhoWDQk9PdKnpiiOHyne5jAVliTwhsDcKYCaqt9F8AAAOyuRkWobU32YpIEnU=" + install: - travis_retry pip install -r requirements.txt - travis_retry pip install flake8 + script: - flake8 --statistics shipyard.py convoy/*.py - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8 --statistics cascade/*.py cargo/*.py; fi + +after_success: +- | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then + echo "Not building binary with PR $TRAVIS_PULL_REQUEST" + return + fi + if [[ "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then + echo "Not building binary without Python 3.6" + return + fi + if [[ ! -z "$TRAVIS_TAG" ]]; then + export UPLOAD_PATH="releases/$TRAVIS_TAG" + elif [[ "$TRAVIS_BRANCH" == "master" ]] || [[ "$TRAVIS_BRANCH" == "develop" ]]; then + export BATCH_SHIPYARD_CLI_ARTIFACT="batch-shipyard-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER}-cli-$TRAVIS_OS_NAME-x86_64" + export UPLOAD_PATH="builds/$TRAVIS_BRANCH" + else + echo "Invalid tag=$TRAVIS_TAG or branch=$TRAVIS_BRANCH to build binary" + return + fi + echo "BATCH_SHIPYARD_CLI_ARTIFACT=$BATCH_SHIPYARD_CLI_ARTIFACT UPLOAD_PATH=$UPLOAD_PATH" + if [[ ! -e "bin/$BATCH_SHIPYARD_CLI_ARTIFACT" ]]; then + travis_retry pip install --upgrade --pre blobxfer + virtualenv -p python3 pyi + $SHELL -c "set -e; source pyi/bin/activate; \ + pip install pyinstaller; \ + pip install --no-cache-dir -r requirements.txt; \ + pyinstaller -F -n $BATCH_SHIPYARD_CLI_ARTIFACT -p batch-shipyard --add-data scripts:scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py; \ + deactivate" + blobxfer upload --remote-path $UPLOAD_PATH --local-path bin/$BATCH_SHIPYARD_CLI_ARTIFACT --strip-components 1 --file-md5 --overwrite + fi + +deploy: + - provider: releases + skip_cleanup: true + on: + tags: true + condition: "$TRAVIS_PYTHON_VERSION == 3.6" + draft: true + overwrite: true + tag_name: $TRAVIS_TAG + file: bin/$BATCH_SHIPYARD_CLI_ARTIFACT + api_key: + secure: Mz2sWn3YAr4tenO1NflOR6QJ9xiy4bjQvnFl9HzZ4zifxrjwLyN/Nbms6X2l8lq7+HTIDOn4Nw8DsE60Upq7yG0UEkwv0BJJtl2qzPnWGOAymykjnTP5Vl5MlCcqEVtUmWL+Wl7WMid10eJLdxSo7TUHp/AW/NShJuKj4rqQsdOtWBpWHuE2rBVTYeuPpUVku74rGH813erxgX/BPzcSjWcloha3Zp25jglJQ4sjqtGrmdts3hXYI1/6oEvdnmFn2xePRMfrEhJxkHkGhqEukkehi4QMnxQubcv9Y9ETQTROf0QjtTeJ+h/HsK2wY5VdI4dcqe3b0/F0m5sZ2Z0PThzBoi/ga18OKeU8P/w1xWk2+DtvG9jGZ8yjbnfRLxOfEg8KNF8Y6B6KntkJWpoNBlIdDU9jOndnvdAjFu90KMmgerz53FtLw5DmZQxIUjoryRrFQMjnvR2dbaP8QF2LlbdlSPDXcGSlO4zVh9BxXFQJgJ9PYA9AQnVii32o1RL5+CelV59AKaheybtk2pqUVvF1mz+prPVbsIJOPo97YpaAfuNprqVvE8kfE9t5W7aOhMuqM+bS4LpENzqDsPIro2cJ2FROtFmB9Ts1YTt6P0UK0XY+m2v3URWjw8v7U4IqGPeNMdQUC6wEJgNxPH1Ko+T1ZMhQ9y94FWO2at0wF3Q= diff --git a/CHANGELOG.md b/CHANGELOG.md index 41e5372..0febe29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Added - Singularity support - Preliminary Windows server support +- Pre-built binaries for CLI for some Linux distributions and Windows +- Windows Docker image for CLI - Singularity HPCG and TensorFlow-GPU recipe ### Changed diff --git a/README.md b/README.md index 053d4f5..79eaee9 100644 --- a/README.md +++ b/README.md @@ -97,9 +97,6 @@ required. Simply request a Cloud Shell session and type `shipyard` to invoke the CLI. ### Local Installation -Installation is typically an easy two-step process. The CLI is also available -as a Docker image: -[alfpark/batch-shipyard:latest-cli](https://hub.docker.com/r/alfpark/batch-shipyard). Please see [the installation guide](http://batch-shipyard.readthedocs.io/en/latest/01-batch-shipyard-installation/) for more information regarding installation and requirements. diff --git a/appveyor.yml b/appveyor.yml index d05fcdd..1f81236 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,6 +15,10 @@ environment: PYTHON: "C:\\Python36-x64" PYTHON_VERSION: "3.6" PYTHON_ARCH: "64" + BLOBXFER_STORAGE_ACCOUNT_KEY: + secure: kc+BqqjCZH3FGmN+4ngtNa8p4p2CR93gTGB9ecXvAZBdeOC3aaQoAV1JSSxXemfYF1I8B8rWM6gp6YyFusqfkpH4KMTZAyI8pTzV9l8dDaKhTBfE21kYlXgsRpV+JUnj + BLOBXFER_STORAGE_ACCOUNT: + secure: Dlu9Wk8y1Mo4eGjx6cH6ex+35JEB2g7kpsK15bSZeAI= DOCKER_USERNAME: secure: S8n3Geq7JUkN7ZQKXo8CLg== DOCKER_PASSWORD: @@ -46,31 +50,104 @@ after_test: ) - ps: >- if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { - Write-Host "Build is from a PR, not creating a Docker image" + Write-Host "Build is from a PR, not creating binary" return } if (!($env:PYTHON_VERSION -eq "3.6")) { - Write-Host "Python environment is not 3.6, not creating a Docker image" + Write-Host "Python environment is not 3.6, not creating binary" + return + } + if ($env:APPVEYOR_REPO_TAG -eq "true") { + $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_TAG_NAME + "-cli-win-amd64.exe" + $env:UPLOAD_PATH="releases/" + $env:APPVEYOR_REPO_TAG_NAME + } + else { + if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop") { + $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_BRANCH + "-" + $env:APPVEYOR_BUILD_NUMBER + "-cli-win-amd64.exe" + $env:UPLOAD_PATH="builds/" + $env:APPVEYOR_REPO_BRANCH + } + else { + Write-Host "Invalid tag or branch $env:APPVEYOR_REPO_BRANCH to build binary" + return + } + } +- IF "%BATCH_SHIPYARD_CLI_ARITFACT%"=="" ( + echo "Batch Shipyard CLI artifact not defined" + ) ELSE ( + echo "Batch Shipyard CLI is %BATCH_SHIPYARD_CLI_ARITFACT% upload path %UPLOAD_PATH%" && + pip install --upgrade --pre blobxfer && + virtualenv -p "%PYTHON%\\python.exe" pyi && + pyi\\Scripts\\activate.bat && + pip install --upgrade pyinstaller && + pip install -r requirements.txt && + pyinstaller -F -n "%BATCH_SHIPYARD_CLI_ARITFACT%" -p batch-shipyard --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py && + pyi\\Scripts\\deactivate.bat && + appveyor PushArtifact "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%" && + blobxfer upload --remote-path %UPLOAD_PATH% --local-path bin\\%BATCH_SHIPYARD_CLI_ARITFACT% --strip-components 1 --file-md5 --overwrite + ) +- ps: >- + if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { + Write-Host "Build is from a PR, not creating a Docker image for cargo" + return + } + if (!($env:PYTHON_VERSION -eq "3.6")) { + Write-Host "Python environment is not 3.6, not creating a Docker image for cargo" return } $DOCKER_IMAGE_TAG_SUFFIX = $null if ($env:APPVEYOR_REPO_TAG -eq "true") { - $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-windows' + $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cargo-windows' } if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) { - Write-Host "Image tag suffix is null, not creating a Docker image" + Write-Host "Image tag suffix is null, not creating a Docker image for cargo" return } $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX - Write-Host "Creating image with tag: $DOCKER_IMAGE_TAG" + Write-Host "Creating Docker image for cargo with tag: $DOCKER_IMAGE_TAG" docker version - pushd cargo\\windows + pushd docker\\windows\\cargo + + docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG . + + docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD" + + docker push $DOCKER_IMAGE_TAG + + popd + +- ps: >- + if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) { + Write-Host "Build is from a PR, not creating a Docker image for CLI" + return + } + if (!($env:PYTHON_VERSION -eq "3.6")) { + Write-Host "Python environment is not 3.6, not creating a Docker image for CLI" + return + } + + $DOCKER_IMAGE_TAG_SUFFIX = $null + + if ($env:APPVEYOR_REPO_TAG -eq "true") { + $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cli-windows' + } + if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) { + Write-Host "Image tag suffix is null, not creating a Docker image for CLI" + return + } + + $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX + + Write-Host "Creating Docker image for CLI with tag: $DOCKER_IMAGE_TAG" + + docker version + + pushd docker\\windows\\cli docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG . diff --git a/cargo/windows/recurrent_job_manager.cmd b/cargo/recurrent_job_manager.cmd similarity index 100% rename from cargo/windows/recurrent_job_manager.cmd rename to cargo/recurrent_job_manager.cmd diff --git a/docker/linux/cli/Dockerfile b/docker/linux/cli/Dockerfile index b6a86dd..8e1d8ee 100644 --- a/docker/linux/cli/Dockerfile +++ b/docker/linux/cli/Dockerfile @@ -14,7 +14,7 @@ RUN apk update \ && cd /opt/batch-shipyard \ && git checkout $GIT_COMMIT \ && rm -rf .git \ - && rm -f .git* .travis.yml install* \ + && rm -f .git* .travis.yml appveyor.yml install* \ && pip3 install --no-cache-dir -r requirements.txt \ && python3 -m compileall -f /opt/batch-shipyard \ && apk del --purge build-base python3-dev libressl-dev libffi-dev git \ diff --git a/cargo/windows/Dockerfile b/docker/windows/cargo/Dockerfile similarity index 94% rename from cargo/windows/Dockerfile rename to docker/windows/cargo/Dockerfile index e364f35..34d38b9 100644 --- a/cargo/windows/Dockerfile +++ b/docker/windows/cargo/Dockerfile @@ -16,8 +16,8 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch --depth 5 https://github.com/Az git checkout $Env:GIT_COMMIT ; \ pip install --no-cache-dir -r cargo\requirements.txt ; \ del C:\batch-shipyard\cargo\*.sh ; \ - del C:\batch-shipyard\cargo\Dockerfile ; \ - copy C:\batch-shipyard\cargo\windows\*.cmd C:\batch-shipyard\cargo + del C:\batch-shipyard\cargo\requirements.txt ; \ + del C:\batch-shipyard\cargo\Dockerfile RUN python -m compileall C:\Python\Lib ; \ python -m compileall C:\batch-shipyard\cargo ; \ diff --git a/docker/windows/cli/Dockerfile b/docker/windows/cli/Dockerfile new file mode 100644 index 0000000..a89b6f5 --- /dev/null +++ b/docker/windows/cli/Dockerfile @@ -0,0 +1,50 @@ +# Dockerfile for Azure/batch-shipyard CLI (Windows) +# Adapted from: https://github.com/StefanScherer/dockerfiles-windows/blob/master/python/Dockerfile + +FROM python:3.6.3-windowsservercore +MAINTAINER Fred Park + +ENV chocolateyUseWindowsCompression false +RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \ + choco install --no-progress -y git -params "/GitAndUnixToolsOnPath" + +ARG GIT_BRANCH +ARG GIT_COMMIT + +WORKDIR C:\\batch-shipyard +RUN git clone -b $Env:GIT_BRANCH --single-branch --depth 5 https://github.com/Azure/batch-shipyard.git C:\batch-shipyard ; \ + git checkout $Env:GIT_COMMIT ; \ + pip install --no-cache-dir -r requirements.txt ; \ + Remove-Item .git -Force -Recurse ; \ + Remove-Item .git* -Force -Recurse ; \ + Remove-Item .travis.yml -Force -Recurse ; \ + Remove-Item appveyor.yml -Force -Recurse ; \ + Remove-Item install* -Force -Recurse + +RUN python -m compileall C:\Python\Lib ; \ + python -m compileall C:\batch-shipyard ; \ + exit 0 + +FROM microsoft/nanoserver + +COPY --from=0 /Python /Python +COPY --from=0 /batch-shipyard /batch-shipyard + +SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] + +ENV PYTHON_VERSION 3.6.3 +ENV PYTHON_PIP_VERSION 9.0.1 + +RUN $env:PATH = 'C:\Python;C:\Python\Scripts;{0}' -f $env:PATH ; \ + Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $env:PATH ; \ + mkdir $env:APPDATA\Python\Python36\site-packages ; \ + Invoke-WebRequest 'https://bootstrap.pypa.io/get-pip.py' -OutFile 'get-pip.py' -UseBasicParsing ; \ + $replace = ('import tempfile{0}import site{0}site.getusersitepackages()' -f [char][int]10) ; \ + Get-Content get-pip.py | Foreach-Object { $_ -replace 'import tempfile', $replace } | Out-File -Encoding Ascii getpip.py ; \ + $pipInstall = ('pip=={0}' -f $env:PYTHON_PIP_VERSION) ; \ + python getpip.py $pipInstall ; \ + Remove-Item get-pip.py ; \ + Remove-Item getpip.py + +WORKDIR C:\\batch-shipyard +ENTRYPOINT ["python", "shipyard.py"] diff --git a/docs/01-batch-shipyard-installation.md b/docs/01-batch-shipyard-installation.md index b7c04a0..6bde292 100644 --- a/docs/01-batch-shipyard-installation.md +++ b/docs/01-batch-shipyard-installation.md @@ -1,12 +1,18 @@ # Batch Shipyard Installation -Installation is an easy two-step process if not using Azure Cloud Shell: -fetch the code and run the install script to download and setup dependencies. +There are multiple available options for installing Batch Shipyard. Please +pick an option that is most suitable for your work environment. + +* [Azure Cloud Shell](#cloudshell) +* [Pre-built binary](#binary) +* [Installers](#installers) +* [Docker image](#docker-install) +* [Jupyter Notebooks](#jupyter) If you wish to install Batch Shipyard into your Azure App Service (e.g., Azure Function App) environment, please see [this guide](60-batch-shipyard-site-extension.md). -### Azure Cloud Shell +## Azure Cloud Shell Batch Shipyard is now integrated into [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview) with no installation required. Simply request a Cloud Shell session and type @@ -20,12 +26,21 @@ the command `shipyard --version`. If you wish to install Batch Shipyard on your machine, please proceed to the Installation section. -### Jupyter Notebooks -There are community contributed [Jupyter notebooks](../contrib/notebooks) to -help you quickly get started if you prefer that environment instead of a -commandline. +## Pre-built Binary +Download an appropriate [Release](https://github.com/Azure/batch-shipyard/releases) +binary for your operating system. Pre-built binaries are not available +for all platforms and architectures at this time. + +Note that for the Linux pre-built binary, it may not work on all +distributions. If this is the case, please pick an alternate installation +method. + +## Installation via Script +Installation is an easy two-step process if using the installers: fetch the +code and run the install script to download and setup dependencies. This +is typically the most flexible and compatible installation outside of the +Docker image for the CLI. -## Installation ### Step 1: Acquire Batch Shipyard Clone the repository: ```shell @@ -305,6 +320,11 @@ You are now ready to execute it with `docker run`. Please see the [Batch Shipyard Usage](20-batch-shipyard-usage.md) guide for more information on how to execute the Batch Shipyard CLI Docker image. +## Jupyter Notebooks +There are community contributed [Jupyter notebooks](../contrib/notebooks) to +help you quickly get started if you prefer that environment instead of a +commandline. + ## Upgrading to New Releases To upgrade to a new release, simply execute `git pull` or download a new release archive and unpack. Next, upgrade the dependencies for your @@ -323,6 +343,9 @@ Rerun the `install.cmd` script with the same virtual environment parameter. If using the CLI Docker image, simply re-issue the `docker pull` command above. +#### Pre-built Binary +Download a new version of the binary. + ## Windows Support Please note that while Batch Shipyard can run on Windows, some functionality may not be supported in Windows out of the box such as SSH, scp, rsync, and diff --git a/shipyard.py b/shipyard.py index 0b87301..c4afacc 100755 --- a/shipyard.py +++ b/shipyard.py @@ -683,7 +683,8 @@ def fs_cluster_options(f): @click.version_option(version=convoy.__version__) @click.pass_context def cli(ctx): - """Batch Shipyard: Provision and Execute Docker Workloads on Azure Batch""" + """Batch Shipyard: Provision and execute container workloads on + Azure Batch""" pass