Generate pre-built binaries, Windows docker images

- Pre-built CLI binaries for Linux x86_64, Windows amd64 - Docker CLI image for Windows
2017-11-05 14:06:52 -08:00 · 2017-11-05 14:06:52 -08:00 · f9936fc5e2
--- a/.travis.yml
+++ b/.travis.yml
@ -1,13 +1,67 @@
 language: python
+
 cache: pip
+
 python:
  - 2.7
  - 3.4
  - 3.5
  - 3.6
+
+env:
+  global:
+  - BATCH_SHIPYARD_CLI_ARTIFACT=batch-shipyard-$TRAVIS_TAG-cli-$TRAVIS_OS_NAME-x86_64
+  - secure: "CsiojO04pOwYbCduqXcZfSsJZ01n0bc4hflza4tk9ZHLotcZkCRKGVsemdPVKZVNlzy+gUpzyc4Tc4/h0391piQ51CCTThde2csDWbXssoO71b4U/jIuKhOqMHyp+5dGznBXOcFye9d5gpdHF0z3AZbo9V7zkuO+3YD/ES1YLWJ5+6RRAR11FRofipfNFYQ7OPgPzrqI0IHHKOYG02JeN7ZmpX+fEitH/J561y9RMIhEtk2udBd51JSTIYao7IENEpHoD41CwwlpOLWKhr7oo9wmyOTdvwyNzLh6Z/CxBUMvmvKn4zsK2d/WzM77EwN4bcBwu2vJmfR+Hi5CD61L9KdhVM+rWUn8+OJUmiNo+ZYVM5w/cHdyC7BYe5hHDAEVmvg0dQIxFXTov5+PjAi0HjA2RNrI2i2ABX1N8wrtPCxlkL8Pnbb93DfTPWmYQVS6LsXrh77KuD7gRxu1s5KAmlfcG//bGuKvpDz6vm7+/cXxnB4sZX+Et9S/F1GV1VYOAr3YwLuE2Tl8MPyoUJa/PdK+md3klL9WteQWXJHLcLbnt34VvstF0jxGh/hqI/3lKtN1U5bsfRbGNBAvYGbGQKpVleF5/0yDLI5/kWI7bIuEF4FHYgIOSvIAzVdC+GXYAeXgrjlOHDFVKwOGL1lhvY3nA/fbENNhHeEN3RQc6N8="
+  - secure: "XgD2xFqMvimEDzwRvUwNmA7EAl6Y/ydwgXXq+O4eEE09Xl/UQvmCZ4soTwFqOUnf798Xqn6R2qXHdqpl2l4IXMILa4LteNUzRKd0txOj2lPGftFs1s09Odu729PkoEs9qbM446yjvK+PhbDUvD9GboQ3sWbzYSUPPDiJuPWiJ5KasooGdyfahV542hBo2iOc0Ezq+sziBwBffY407XcX7mTmXjEtQuwSezJGWynHUSD7gIywCXAuzTqdK3ikYYdSJTUn6SnlZZlLuICoOzVULT97VzA+o+puFU/rSnG9+czUUcXILpDzjpl231b858sQ7RdLocPdTiyIVx06FOfW/XfKUyDbRHGhiVZzsJ4xDMxtQk9haLb5NJjwZGFWmyy2tHnj86yNwxwmocbiRymqC2sz/rcIpmwMneKIGJG9MuXRMlGqeMxCcDb3FoaoUp6JIflkx5qsmgsaTEs4m0tcdyivf6ugW1dpBIGek8sf/LISmdakZTNeJxGUyMNYRFdSRWTTZvQ4fj3ZVxLoTCHLszwwHZ1l3AstbiYk86/5Zn8AS+5cWnVEZP3ayPkWqVKMjLxQrHQsLKGgpJBjDXTUp4NixssgXWmDmJMuME0TEjoT58hhoWDQk9PdKnpiiOHyne5jAVliTwhsDcKYCaqt9F8AAAOyuRkWobU32YpIEnU="
+
 install:
  - travis_retry pip install -r requirements.txt
  - travis_retry pip install flake8
+
 script:
  - flake8 --statistics shipyard.py convoy/*.py
  - if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then flake8 --statistics cascade/*.py cargo/*.py; fi
+
+after_success:
+- |
+    if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
+      echo "Not building binary with PR $TRAVIS_PULL_REQUEST"
+      return
+    fi
+    if [[ "$TRAVIS_PYTHON_VERSION" != "3.6" ]]; then
+      echo "Not building binary without Python 3.6"
+      return
+    fi
+    if [[ ! -z "$TRAVIS_TAG" ]]; then
+      export UPLOAD_PATH="releases/$TRAVIS_TAG"
+    elif [[ "$TRAVIS_BRANCH" == "master" ]] || [[ "$TRAVIS_BRANCH" == "develop" ]]; then
+      export BATCH_SHIPYARD_CLI_ARTIFACT="batch-shipyard-${TRAVIS_BRANCH}-${TRAVIS_BUILD_NUMBER}-cli-$TRAVIS_OS_NAME-x86_64"
+      export UPLOAD_PATH="builds/$TRAVIS_BRANCH"
+    else
+      echo "Invalid tag=$TRAVIS_TAG or branch=$TRAVIS_BRANCH to build binary"
+      return
+    fi
+    echo "BATCH_SHIPYARD_CLI_ARTIFACT=$BATCH_SHIPYARD_CLI_ARTIFACT UPLOAD_PATH=$UPLOAD_PATH"
+    if [[ ! -e "bin/$BATCH_SHIPYARD_CLI_ARTIFACT" ]]; then
+      travis_retry pip install --upgrade --pre blobxfer
+      virtualenv -p python3 pyi
+      $SHELL -c "set -e; source pyi/bin/activate; \
+                 pip install pyinstaller; \
+                 pip install --no-cache-dir -r requirements.txt; \
+                 pyinstaller -F -n $BATCH_SHIPYARD_CLI_ARTIFACT -p batch-shipyard --add-data scripts:scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py; \
+                 deactivate"
+      blobxfer upload --remote-path $UPLOAD_PATH --local-path bin/$BATCH_SHIPYARD_CLI_ARTIFACT --strip-components 1 --file-md5 --overwrite
+    fi
+
+deploy:
+  - provider: releases
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$TRAVIS_PYTHON_VERSION == 3.6"
+    draft: true
+    overwrite: true
+    tag_name: $TRAVIS_TAG
+    file: bin/$BATCH_SHIPYARD_CLI_ARTIFACT
+    api_key:
+      secure: Mz2sWn3YAr4tenO1NflOR6QJ9xiy4bjQvnFl9HzZ4zifxrjwLyN/Nbms6X2l8lq7+HTIDOn4Nw8DsE60Upq7yG0UEkwv0BJJtl2qzPnWGOAymykjnTP5Vl5MlCcqEVtUmWL+Wl7WMid10eJLdxSo7TUHp/AW/NShJuKj4rqQsdOtWBpWHuE2rBVTYeuPpUVku74rGH813erxgX/BPzcSjWcloha3Zp25jglJQ4sjqtGrmdts3hXYI1/6oEvdnmFn2xePRMfrEhJxkHkGhqEukkehi4QMnxQubcv9Y9ETQTROf0QjtTeJ+h/HsK2wY5VdI4dcqe3b0/F0m5sZ2Z0PThzBoi/ga18OKeU8P/w1xWk2+DtvG9jGZ8yjbnfRLxOfEg8KNF8Y6B6KntkJWpoNBlIdDU9jOndnvdAjFu90KMmgerz53FtLw5DmZQxIUjoryRrFQMjnvR2dbaP8QF2LlbdlSPDXcGSlO4zVh9BxXFQJgJ9PYA9AQnVii32o1RL5+CelV59AKaheybtk2pqUVvF1mz+prPVbsIJOPo97YpaAfuNprqVvE8kfE9t5W7aOhMuqM+bS4LpENzqDsPIro2cJ2FROtFmB9Ts1YTt6P0UK0XY+m2v3URWjw8v7U4IqGPeNMdQUC6wEJgNxPH1Ko+T1ZMhQ9y94FWO2at0wF3Q=
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,8 @@
 ### Added
 - Singularity support
 - Preliminary Windows server support
+- Pre-built binaries for CLI for some Linux distributions and Windows
+- Windows Docker image for CLI
 - Singularity HPCG and TensorFlow-GPU recipe

 ### Changed
--- a/README.md
+++ b/README.md
@ -97,9 +97,6 @@ required. Simply request a Cloud Shell session and type `shipyard` to invoke
 the CLI.

 ### Local Installation
-Installation is typically an easy two-step process. The CLI is also available
-as a Docker image:
-[alfpark/batch-shipyard:latest-cli](https://hub.docker.com/r/alfpark/batch-shipyard).
 Please see [the installation guide](http://batch-shipyard.readthedocs.io/en/latest/01-batch-shipyard-installation/)
 for more information regarding installation and requirements.

--- a/appveyor.yml
+++ b/appveyor.yml
@ -15,6 +15,10 @@ environment:
    PYTHON: "C:\\Python36-x64"
    PYTHON_VERSION: "3.6"
    PYTHON_ARCH: "64"
+  BLOBXFER_STORAGE_ACCOUNT_KEY:
+    secure: kc+BqqjCZH3FGmN+4ngtNa8p4p2CR93gTGB9ecXvAZBdeOC3aaQoAV1JSSxXemfYF1I8B8rWM6gp6YyFusqfkpH4KMTZAyI8pTzV9l8dDaKhTBfE21kYlXgsRpV+JUnj
+  BLOBXFER_STORAGE_ACCOUNT:
+    secure: Dlu9Wk8y1Mo4eGjx6cH6ex+35JEB2g7kpsK15bSZeAI=
  DOCKER_USERNAME:
    secure: S8n3Geq7JUkN7ZQKXo8CLg==
  DOCKER_PASSWORD:
@ -46,31 +50,104 @@ after_test:
  )
 - ps: >-
    if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) {
-      Write-Host "Build is from a PR, not creating a Docker image"
+      Write-Host "Build is from a PR, not creating binary"
      return
    }
    if (!($env:PYTHON_VERSION -eq "3.6")) {
-      Write-Host "Python environment is not 3.6, not creating a Docker image"
+      Write-Host "Python environment is not 3.6, not creating binary"
+      return
+    }
+    if ($env:APPVEYOR_REPO_TAG -eq "true") {
+      $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_TAG_NAME + "-cli-win-amd64.exe"
+      $env:UPLOAD_PATH="releases/" + $env:APPVEYOR_REPO_TAG_NAME
+    }
+    else {
+      if ($env:APPVEYOR_REPO_BRANCH -eq "master" -Or $env:APPVEYOR_REPO_BRANCH -eq "develop") {
+        $env:BATCH_SHIPYARD_CLI_ARITFACT = "batch-shipyard-" + $env:APPVEYOR_REPO_BRANCH + "-" + $env:APPVEYOR_BUILD_NUMBER + "-cli-win-amd64.exe"
+        $env:UPLOAD_PATH="builds/" + $env:APPVEYOR_REPO_BRANCH
+      }
+      else {
+        Write-Host "Invalid tag or branch $env:APPVEYOR_REPO_BRANCH to build binary"
+        return
+      }
+    }
+- IF "%BATCH_SHIPYARD_CLI_ARITFACT%"=="" (
+    echo "Batch Shipyard CLI artifact not defined"
+  ) ELSE (
+    echo "Batch Shipyard CLI is %BATCH_SHIPYARD_CLI_ARITFACT% upload path %UPLOAD_PATH%" &&
+    pip install --upgrade --pre blobxfer &&
+    virtualenv -p "%PYTHON%\\python.exe" pyi &&
+    pyi\\Scripts\\activate.bat &&
+    pip install --upgrade pyinstaller &&
+    pip install -r requirements.txt &&
+    pyinstaller -F -n "%BATCH_SHIPYARD_CLI_ARITFACT%" -p batch-shipyard --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --distpath bin shipyard.py &&
+    pyi\\Scripts\\deactivate.bat &&
+    appveyor PushArtifact "bin\\%BATCH_SHIPYARD_CLI_ARITFACT%" &&
+    blobxfer upload --remote-path %UPLOAD_PATH% --local-path bin\\%BATCH_SHIPYARD_CLI_ARITFACT% --strip-components 1 --file-md5 --overwrite
+  )
+- ps: >-
+    if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) {
+      Write-Host "Build is from a PR, not creating a Docker image for cargo"
+      return
+    }
+    if (!($env:PYTHON_VERSION -eq "3.6")) {
+      Write-Host "Python environment is not 3.6, not creating a Docker image for cargo"
      return
    }

    $DOCKER_IMAGE_TAG_SUFFIX = $null

    if ($env:APPVEYOR_REPO_TAG -eq "true") {
-      $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-windows'
+      $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cargo-windows'
    }
    if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) {
-      Write-Host "Image tag suffix is null, not creating a Docker image"
+      Write-Host "Image tag suffix is null, not creating a Docker image for cargo"
      return
    }

    $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX

-    Write-Host "Creating image with tag: $DOCKER_IMAGE_TAG"
+    Write-Host "Creating Docker image for cargo with tag: $DOCKER_IMAGE_TAG"

    docker version

-    pushd cargo\\windows
+    pushd docker\\windows\\cargo
+
+    docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG .
+
+    docker login -u="$env:DOCKER_USERNAME" -p="$env:DOCKER_PASSWORD"
+
+    docker push $DOCKER_IMAGE_TAG
+
+    popd
+
+- ps: >-
+    if (!($env:APPVEYOR_PULL_REQUEST_NUMBER -eq $null)) {
+      Write-Host "Build is from a PR, not creating a Docker image for CLI"
+      return
+    }
+    if (!($env:PYTHON_VERSION -eq "3.6")) {
+      Write-Host "Python environment is not 3.6, not creating a Docker image for CLI"
+      return
+    }
+
+    $DOCKER_IMAGE_TAG_SUFFIX = $null
+
+    if ($env:APPVEYOR_REPO_TAG -eq "true") {
+      $DOCKER_IMAGE_TAG_SUFFIX = $env:APPVEYOR_REPO_TAG_NAME + '-cli-windows'
+    }
+    if ($DOCKER_IMAGE_TAG_SUFFIX -eq $null) {
+      Write-Host "Image tag suffix is null, not creating a Docker image for CLI"
+      return
+    }
+
+    $DOCKER_IMAGE_TAG = $env:DOCKER_IMAGE_TAG_PREFIX + $DOCKER_IMAGE_TAG_SUFFIX
+
+    Write-Host "Creating Docker image for CLI with tag: $DOCKER_IMAGE_TAG"
+
+    docker version
+
+    pushd docker\\windows\\cli

    docker build --build-arg GIT_BRANCH=$env:APPVEYOR_REPO_BRANCH --build-arg GIT_COMMIT=$env:APPVEYOR_REPO_COMMIT -t $DOCKER_IMAGE_TAG .

--- a/cargo/windows/recurrent_job_manager.cmd
+++ b/cargo/windows/recurrent_job_manager.cmd
--- a/docker/linux/cli/Dockerfile
+++ b/docker/linux/cli/Dockerfile
@ -14,7 +14,7 @@ RUN apk update \
    && cd /opt/batch-shipyard \
    && git checkout $GIT_COMMIT \
    && rm -rf .git \
-    && rm -f .git* .travis.yml install* \
+    && rm -f .git* .travis.yml appveyor.yml install* \
    && pip3 install --no-cache-dir -r requirements.txt \
    && python3 -m compileall -f /opt/batch-shipyard \
    && apk del --purge build-base python3-dev libressl-dev libffi-dev git \
--- a/docker/windows/cargo/Dockerfile
+++ b/docker/windows/cargo/Dockerfile
@ -16,8 +16,8 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch --depth 5 https://github.com/Az
    git checkout $Env:GIT_COMMIT ; \
    pip install --no-cache-dir -r cargo\requirements.txt ; \
 	del C:\batch-shipyard\cargo\*.sh ; \
-	del C:\batch-shipyard\cargo\Dockerfile ; \
-    copy C:\batch-shipyard\cargo\windows\*.cmd C:\batch-shipyard\cargo
+	del C:\batch-shipyard\cargo\requirements.txt ; \
+	del C:\batch-shipyard\cargo\Dockerfile

 RUN python -m compileall C:\Python\Lib ; \
    python -m compileall C:\batch-shipyard\cargo ; \
--- a/docker/windows/cli/Dockerfile
+++ b/docker/windows/cli/Dockerfile
@ -0,0 +1,50 @@
+# Dockerfile for Azure/batch-shipyard CLI (Windows)
+# Adapted from: https://github.com/StefanScherer/dockerfiles-windows/blob/master/python/Dockerfile
+
+FROM python:3.6.3-windowsservercore
+MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
+
+ENV chocolateyUseWindowsCompression false
+RUN iex ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')); \
+    choco install --no-progress -y git -params "/GitAndUnixToolsOnPath"
+
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+
+WORKDIR C:\\batch-shipyard
+RUN git clone -b $Env:GIT_BRANCH --single-branch --depth 5 https://github.com/Azure/batch-shipyard.git C:\batch-shipyard ; \
+    git checkout $Env:GIT_COMMIT ; \
+    pip install --no-cache-dir -r requirements.txt ; \
+    Remove-Item .git -Force -Recurse ; \
+    Remove-Item .git* -Force -Recurse ; \
+    Remove-Item .travis.yml -Force -Recurse ; \
+    Remove-Item appveyor.yml -Force -Recurse ; \
+    Remove-Item install* -Force -Recurse
+
+RUN python -m compileall C:\Python\Lib ; \
+    python -m compileall C:\batch-shipyard ; \
+	exit 0
+
+FROM microsoft/nanoserver
+
+COPY --from=0 /Python /Python
+COPY --from=0 /batch-shipyard /batch-shipyard
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+ENV PYTHON_VERSION 3.6.3
+ENV PYTHON_PIP_VERSION 9.0.1
+
+RUN $env:PATH = 'C:\Python;C:\Python\Scripts;{0}' -f $env:PATH ; \
+    Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\Environment\' -Name Path -Value $env:PATH ; \
+    mkdir $env:APPDATA\Python\Python36\site-packages ; \
+    Invoke-WebRequest 'https://bootstrap.pypa.io/get-pip.py' -OutFile 'get-pip.py' -UseBasicParsing ; \
+    $replace = ('import tempfile{0}import site{0}site.getusersitepackages()' -f [char][int]10) ; \
+    Get-Content get-pip.py | Foreach-Object { $_ -replace 'import tempfile', $replace } | Out-File -Encoding Ascii getpip.py ; \
+    $pipInstall = ('pip=={0}' -f $env:PYTHON_PIP_VERSION) ; \
+    python getpip.py $pipInstall ; \
+    Remove-Item get-pip.py ; \
+    Remove-Item getpip.py
+
+WORKDIR C:\\batch-shipyard
+ENTRYPOINT ["python", "shipyard.py"]
--- a/docs/01-batch-shipyard-installation.md
+++ b/docs/01-batch-shipyard-installation.md
@ -1,12 +1,18 @@
 # Batch Shipyard Installation
-Installation is an easy two-step process if not using Azure Cloud Shell:
-fetch the code and run the install script to download and setup dependencies.
+There are multiple available options for installing Batch Shipyard. Please
+pick an option that is most suitable for your work environment.
+
+* [Azure Cloud Shell](#cloudshell)
+* [Pre-built binary](#binary)
+* [Installers](#installers)
+* [Docker image](#docker-install)
+* [Jupyter Notebooks](#jupyter)

 If you wish to install Batch Shipyard into your Azure App Service (e.g.,
 Azure Function App) environment, please see
 [this guide](60-batch-shipyard-site-extension.md).

-### Azure Cloud Shell
+## <a name="cloudshell"></a>Azure Cloud Shell
 Batch Shipyard is now integrated into
 [Azure Cloud Shell](https://docs.microsoft.com/en-us/azure/cloud-shell/overview)
 with no installation required. Simply request a Cloud Shell session and type
@ -20,12 +26,21 @@ the command `shipyard --version`.
 If you wish to install Batch Shipyard on your machine, please proceed to the
 Installation section.

-### Jupyter Notebooks
-There are community contributed [Jupyter notebooks](../contrib/notebooks) to
-help you quickly get started if you prefer that environment instead of a
-commandline.
+## <a name="binary"></a>Pre-built Binary
+Download an appropriate [Release](https://github.com/Azure/batch-shipyard/releases)
+binary for your operating system. Pre-built binaries are not available
+for all platforms and architectures at this time.
+
+Note that for the Linux pre-built binary, it may not work on all
+distributions. If this is the case, please pick an alternate installation
+method.
+
+## <a name="installers"></a>Installation via Script
+Installation is an easy two-step process if using the installers: fetch the
+code and run the install script to download and setup dependencies. This
+is typically the most flexible and compatible installation outside of the
+Docker image for the CLI.

-## Installation
 ### Step 1: Acquire Batch Shipyard
 Clone the repository:
 ```shell
@ -305,6 +320,11 @@ You are now ready to execute it with `docker run`. Please see the
 [Batch Shipyard Usage](20-batch-shipyard-usage.md) guide for more information
 on how to execute the Batch Shipyard CLI Docker image.

+## <a name="jupyter"></a>Jupyter Notebooks
+There are community contributed [Jupyter notebooks](../contrib/notebooks) to
+help you quickly get started if you prefer that environment instead of a
+commandline.
+
 ## Upgrading to New Releases
 To upgrade to a new release, simply execute `git pull` or download a new
 release archive and unpack. Next, upgrade the dependencies for your
@ -323,6 +343,9 @@ Rerun the `install.cmd` script with the same virtual environment parameter.
 If using the CLI Docker image, simply re-issue the `docker pull` command
 above.

+#### Pre-built Binary
+Download a new version of the binary.
+
 ## Windows Support
 Please note that while Batch Shipyard can run on Windows, some functionality
 may not be supported in Windows out of the box such as SSH, scp, rsync, and
--- a/shipyard.py
+++ b/shipyard.py
@ -683,7 +683,8 @@ def fs_cluster_options(f):
@click.version_option(version=convoy.__version__)
@click.pass_context
 def cli(ctx):
-    """Batch Shipyard: Provision and Execute Docker Workloads on Azure Batch"""
+    """Batch Shipyard: Provision and execute container workloads on
+    Azure Batch"""
    pass