Minor post-release fixes

- Update 3rd party notices
- Move docker directory to build
- Fix old info in a few recipes

.travis.yml

@@ -17,4 +17,4 @@ install:
 script:
   - flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy/*.py
   - if [[ $TRAVIS_PYTHON_VERSION > '3.4' ]]; then flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py slurm/*.py; fi
-  - shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh
+  - shellcheck ./*.sh cargo/*.sh cascade/*.sh images/*.sh scripts/*.sh

.vsts/pipeline.yml

@@ -83,7 +83,7 @@ jobs:
           echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath"
           $branchGitSha1 = [string]::Format("{0}@{1}",$env:BUILD_SOURCEBRANCHNAME,$env:BUILD_SOURCEVERSION.Substring(0,7))
           echo "$env:ARTIFACT_CLI $env:BUILDVER_TUPLE $branchGitSha1"
-          $fileverInfo = "docker\\windows\\file_version_info.txt"
+          $fileverInfo = "images\\docker\\windows\\file_version_info.txt"
           (Get-Content $fileverInfo).replace('{BUILDVER_TUPLE}', $env:BUILDVER_TUPLE) | Set-Content $fileverInfo
           (Get-Content $fileverInfo).replace('{BUILDVER_DOTTED}', $env:BUILDVER_DOTTED) | Set-Content $fileverInfo
           (Get-Content $fileverInfo).replace('{BRANCH_GITSHA1}', $branchGitSha1) | Set-Content $fileverInfo
@@ -106,7 +106,7 @@ jobs:
           pip install --no-cache-dir pyinstaller
           pip install --upgrade -r requirements.txt
           pip install --upgrade --no-deps -r req_nodeps.txt
-          pyinstaller -F -n "%ARTIFACT_CLI%" -p batch-shipyard --add-data federation\\docker-compose.yml;federation --add-data heimdall;heimdall --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon docker\\windows\\azure.ico --version-file docker\\windows\\file_version_info.txt --distpath bin shipyard.py
+          pyinstaller -F -n "%ARTIFACT_CLI%" -p batch-shipyard --add-data federation\\docker-compose.yml;federation --add-data heimdall;heimdall --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon images\\docker\\windows\\azure.ico --version-file images\\docker\\windows\\file_version_info.txt --distpath bin shipyard.py
           blobxfer upload --storage-account "$(blobxfer.storageAccount)" --storage-account-key "$(blobxfer.storageAccountKey)" --remote-path "%ARTIFACT_UPLOAD_PATH%" --local-path "%ARTIFACT_CLI_PATH%" --strip-components 1 --file-md5 --overwrite
           call pyi\\Scripts\\deactivate.bat
         displayName: Build
@@ -146,11 +146,11 @@ jobs:
           docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)"
           $cargoImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cargo-windows"
           $cliImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cli-windows"
-          pushd docker\\windows\\cargo
+          pushd images\\docker\\windows\\cargo
           docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cargoImage .
           docker push $cargoImage
           popd
-          pushd docker\\windows\\cli
+          pushd images\\docker\\windows\\cli
           docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cliImage .
           docker push $cliImage
           popd
@@ -210,7 +210,7 @@ jobs:
           if [[ "$PYENV_VERSION" == 3.7* ]]; then
             flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py slurm/*.py
           fi
-          shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh
+          shellcheck ./*.sh cargo/*.sh cascade/*.sh images/*.sh scripts/*.sh
         displayName: Static Analysis
       - script: |
           set -e
@@ -343,7 +343,7 @@ jobs:
             popd
           fi
           cliImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cli"
-          pushd docker/linux/cli
+          pushd images/docker/linux/cli
           docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cliImage .
           docker push $cliImage
           popd

CHANGELOG.md

@@ -63,6 +63,7 @@ updates to SR-IOV RDMA VM sizes
 - Per-job autoscratch setup failing for more than 2 nodes
 
 ### Removed
+- Peer-to-peer image distribution support
 - Python 3.4 support
 
 ## [3.7.1] - 2019-07-23

THIRD_PARTY_NOTICES.txt

@@ -340,45 +340,34 @@ DEALINGS IN THE SOFTWARE.
 
 click (https://github.com/pallets/click)
 
-Copyright © 2014 by the Pallets team.
+Copyright 2014 Pallets
 
-Some rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
 
-Redistribution and use in source and binary forms of the software as
+1.  Redistributions of source code must retain the above copyright
-well as documentation, with or without modification, are permitted
-provided that the following conditions are met:
-
--   Redistributions of source code must retain the above copyright
     notice, this list of conditions and the following disclaimer.
 
--   Redistributions in binary form must reproduce the above copyright
+2.  Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.
 
--   Neither the name of the copyright holder nor the names of its
+3.  Neither the name of the copyright holder nor the names of its
     contributors may be used to endorse or promote products derived from
     this software without specific prior written permission.
 
-THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
-NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-THIS SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-SUCH DAMAGE.
-
-----
-
-Click uses parts of optparse written by Gregory P. Ward and maintained
-by the Python Software Foundation. This is limited to code in parser.py.
-
-Copyright © 2001-2006 Gregory P. Ward. All rights reserved.
-Copyright © 2002-2006 Python Software Foundation. All rights reserved.
 
 -------------------------------------------------------------------------------
 
@@ -957,6 +946,11 @@ ruamel.yaml (https://bitbucket.org/ruamel/yaml)
 
 singularity (https://github.com/sylabs/singularity)
 
+Copyright (c) 2015-2017, Gregory M. Kurtzer. All rights reserved.
+Copyright (c) 2016-2017, The Regents of the University of California. All right reserved.
+Copyright (c) 2017, SingularityWare, LLC. All rights reserved.
+Copyright (c) 2018-2019, Sylabs, Inc. All rights reserved.
+
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 

docker/libtorrent/1.0.x/Dockerfile

@@ -1,27 +0,0 @@
-# Dockerfile for Azure/batch-shipyard (Cascade libtorrent multi-stage base)
-
-FROM alpine:3.7
-MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
-
-# set environment variables
-ENV libtorrent_version=1.0.11 \
-    CFLAGS=-lstdc++ PYTHON=/usr/bin/python3 PYTHON_VERSION=3.6
-
-# build libtorrent-rasterbar for python3 and cleanup packaging
-RUN apk update \
-    && apk add --update --no-cache \
-        musl build-base python3 python3-dev libressl-dev libffi-dev \
-        boost boost-dev boost-python3 file curl tar bash \
-    && curl -SL https://github.com/arvidn/libtorrent/releases/download/libtorrent-${libtorrent_version//./_}/libtorrent-rasterbar-${libtorrent_version}.tar.gz -o libtorrent-${libtorrent_version}.tar.gz \
-    && tar zxvpf libtorrent-${libtorrent_version}.tar.gz \
-    && cd libtorrent-rasterbar-${libtorrent_version} \
-    && ./configure --prefix=/usr --enable-debug=no --enable-python-binding --with-boost-system=boost_system \
-    && make -j"$(nproc)" install \
-    && ldconfig /usr/lib \
-    && cd .. \
-    && rm -rf libtorrent-rasterbar-${libtorrent_version} \
-    && rm -f libtorrent-${libtorrent_version}.tar.gz \
-    && apk del --purge \
-        build-base python3-dev libressl-dev libffi-dev python boost-dev \
-        file curl \
-    && rm /var/cache/apk/*

docker/libtorrent/1.1.x/Dockerfile

@@ -1,27 +0,0 @@
-# Dockerfile for Azure/batch-shipyard (Cascade libtorrent multi-stage base)
-
-FROM alpine:3.8
-MAINTAINER Fred Park <https://github.com/Azure/batch-shipyard>
-
-# set environment variables
-ENV libtorrent_version=1.1.9 \
-    CFLAGS=-lstdc++ PYTHON=/usr/bin/python3 PYTHON_VERSION=3.6
-
-# build libtorrent-rasterbar for python3 and cleanup packaging
-RUN apk update \
-    && apk add --update --no-cache \
-        musl build-base python3 python3-dev libressl-dev libffi-dev \
-        boost boost-dev boost-python3 file curl tar bash \
-    && curl -SL https://github.com/arvidn/libtorrent/releases/download/libtorrent-${libtorrent_version//./_}/libtorrent-rasterbar-${libtorrent_version}.tar.gz -o libtorrent-${libtorrent_version}.tar.gz \
-    && tar zxvpf libtorrent-${libtorrent_version}.tar.gz \
-    && cd libtorrent-rasterbar-${libtorrent_version} \
-    && ./configure --prefix=/usr --enable-debug=no --enable-python-binding --with-boost-system=boost_system \
-    && make -j"$(nproc)" install \
-    && ldconfig /usr/lib \
-    && cd .. \
-    && rm -rf libtorrent-rasterbar-${libtorrent_version} \
-    && rm -f libtorrent-${libtorrent_version}.tar.gz \
-    && apk del --purge \
-        build-base python3-dev libressl-dev libffi-dev python boost-dev \
-        file curl \
-    && rm /var/cache/apk/*

images/docker/linux/cli/Dockerfile

@@ -13,7 +13,7 @@ RUN apk update \
     && git clone -b $GIT_BRANCH --single-branch https://github.com/Azure/batch-shipyard.git /opt/batch-shipyard \
     && cd /opt/batch-shipyard \
     && git checkout $GIT_COMMIT \
-    && rm -rf .git .github \
+    && rm -rf .git .github .vsts \
     && rm -f .git* .travis.yml *.yml install* \
     && python3 -m pip install --no-cache-dir --upgrade pip \
     && pip3 install --no-cache-dir -r requirements.txt \

images/docker/windows/cli/Dockerfile

@@ -19,6 +19,7 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch https://github.com/Azure/batch-
     pip install --no-cache-dir --no-deps -r req_nodeps.txt ; \
     Remove-Item .git -Force -Recurse ; \
     Remove-Item .git* -Force -Recurse ; \
+    Remove-Item .vsts -Force -Recurse ; \
     Remove-Item .travis.yml -Force -Recurse ; \
     Remove-Item *.yml -Force -Recurse ; \
     Remove-Item install* -Force -Recurse

images/singularity/cli.def

@@ -0,0 +1,22 @@
+# Singularity for Azure/batch-shipyard (cli)
+
+Bootstrap: library
+From: alpine:3.9
+
+%post
+apk update
+apk add --update --no-cache \
+    musl build-base python3 python3-dev openssl-dev libffi-dev \
+    ca-certificates openssl openssh-client rsync git bash
+git clone -b master --single-branch --depth 5 https://github.com/Azure/batch-shipyard.git /opt/batch-shipyard
+cd /opt/batch-shipyard
+rm -rf .git .github .vsts
+rm -f .git* .travis.yml *.yml install*
+pip3 install --no-cache-dir -r requirements.txt
+pip3 install --no-cache-dir --no-deps -r req_nodeps.txt
+python3 -m compileall -f /opt/batch-shipyard/shipyard.py /opt/batch-shipyard/convoy
+apk del --purge build-base python3-dev openssl-dev libffi-dev git
+rm /var/cache/apk/*
+
+%runscript
+exec /opt/batch-shipyard/shipyard.py "$@"

recipes/HPCG-Infiniband-IntelMPI/README.md

@@ -37,8 +37,8 @@ included in the Linpack image.
 The global configuration should set the following properties:
 * `singularity_images` array must have a reference to a valid HPCG image
 that can be run with Intel MPI and Infiniband. This can be
-`shub://alfpark/linpack` which is
+`library://alfpark/linpack/linpack:2018-intel-mkl` which is
-published on [Singularity Hub](https://www.singularity-hub.org/containers/496).
+published on the [Singularity Library](https://cloud.sylabs.io/library/alfpark/linpack/linpack).
 
 ### Jobs Configuration
 #### Docker-based

recipes/HPLinpack-Infiniband-IntelMPI/README.md

@@ -36,8 +36,8 @@ published on [Docker Hub](https://hub.docker.com/r/alfpark/linpack).
 The global configuration should set the following properties:
 * `singularity_images` array must have a reference to a valid HPLinpack image
 that can be run with Intel MPI and Infiniband. This can be
-`shub://alfpark/linpack` which is
+`library://alfpark/linpack/linpack:2018-intel-mkl` which is
-published on [Singularity Hub](https://www.singularity-hub.org/containers/496).
+published on the [Singularity Library](https://cloud.sylabs.io/library/alfpark/linpack/linpack).
 
 ### Jobs Configuration
 #### Docker-based