diff --git a/.travis.yml b/.travis.yml index 469bbb5..5bd86c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ install: script: - flake8 --select F,E,W --ignore W504 --statistics shipyard.py convoy/*.py - if [[ $TRAVIS_PYTHON_VERSION > '3.4' ]]; then flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py slurm/*.py; fi - - shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh + - shellcheck ./*.sh cargo/*.sh cascade/*.sh images/*.sh scripts/*.sh diff --git a/.vsts/pipeline.yml b/.vsts/pipeline.yml index 159ceb1..22d24cb 100644 --- a/.vsts/pipeline.yml +++ b/.vsts/pipeline.yml @@ -83,7 +83,7 @@ jobs: echo "##vso[task.setvariable variable=ARTIFACT_CLI_PATH;]$artifactCliPath" $branchGitSha1 = [string]::Format("{0}@{1}",$env:BUILD_SOURCEBRANCHNAME,$env:BUILD_SOURCEVERSION.Substring(0,7)) echo "$env:ARTIFACT_CLI $env:BUILDVER_TUPLE $branchGitSha1" - $fileverInfo = "docker\\windows\\file_version_info.txt" + $fileverInfo = "images\\docker\\windows\\file_version_info.txt" (Get-Content $fileverInfo).replace('{BUILDVER_TUPLE}', $env:BUILDVER_TUPLE) | Set-Content $fileverInfo (Get-Content $fileverInfo).replace('{BUILDVER_DOTTED}', $env:BUILDVER_DOTTED) | Set-Content $fileverInfo (Get-Content $fileverInfo).replace('{BRANCH_GITSHA1}', $branchGitSha1) | Set-Content $fileverInfo @@ -106,7 +106,7 @@ jobs: pip install --no-cache-dir pyinstaller pip install --upgrade -r requirements.txt pip install --upgrade --no-deps -r req_nodeps.txt - pyinstaller -F -n "%ARTIFACT_CLI%" -p batch-shipyard --add-data federation\\docker-compose.yml;federation --add-data heimdall;heimdall --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon docker\\windows\\azure.ico --version-file docker\\windows\\file_version_info.txt --distpath bin shipyard.py + pyinstaller -F -n "%ARTIFACT_CLI%" -p batch-shipyard --add-data federation\\docker-compose.yml;federation --add-data heimdall;heimdall --add-data schemas;schemas --add-data scripts;scripts --exclude-module future.tests --exclude-module future.backports.test --exclude-module future.moves.test --icon images\\docker\\windows\\azure.ico --version-file images\\docker\\windows\\file_version_info.txt --distpath bin shipyard.py blobxfer upload --storage-account "$(blobxfer.storageAccount)" --storage-account-key "$(blobxfer.storageAccountKey)" --remote-path "%ARTIFACT_UPLOAD_PATH%" --local-path "%ARTIFACT_CLI_PATH%" --strip-components 1 --file-md5 --overwrite call pyi\\Scripts\\deactivate.bat displayName: Build @@ -146,11 +146,11 @@ jobs: docker login "$(docker.servername)" -u="$(docker.username)" -p="$(docker.password)" $cargoImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cargo-windows" $cliImage = "$(docker.servername)/public/azure-batch/shipyard:" + $env:GIT_TAG + "-cli-windows" - pushd docker\\windows\\cargo + pushd images\\docker\\windows\\cargo docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cargoImage . docker push $cargoImage popd - pushd docker\\windows\\cli + pushd images\\docker\\windows\\cli docker build --build-arg GIT_BRANCH=$env:BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$env:BUILD_SOURCEVERSION -t $cliImage . docker push $cliImage popd @@ -210,7 +210,7 @@ jobs: if [[ "$PYENV_VERSION" == 3.7* ]]; then flake8 --select F,E,W --ignore W504 --statistics cascade/*.py cargo/*.py federation/*.py heimdall/*.py slurm/*.py fi - shellcheck ./*.sh cargo/*.sh cascade/*.sh docker/*.sh scripts/*.sh + shellcheck ./*.sh cargo/*.sh cascade/*.sh images/*.sh scripts/*.sh displayName: Static Analysis - script: | set -e @@ -343,7 +343,7 @@ jobs: popd fi cliImage="$(docker.servername)/public/azure-batch/shipyard:${DOCKER_TAG}-cli" - pushd docker/linux/cli + pushd images/docker/linux/cli docker build --build-arg GIT_BRANCH=$BUILD_SOURCEBRANCHNAME --build-arg GIT_COMMIT=$BUILD_SOURCEVERSION -t $cliImage . docker push $cliImage popd diff --git a/CHANGELOG.md b/CHANGELOG.md index 04b9d60..03f102f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ updates to SR-IOV RDMA VM sizes - Per-job autoscratch setup failing for more than 2 nodes ### Removed +- Peer-to-peer image distribution support - Python 3.4 support ## [3.7.1] - 2019-07-23 diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index a859e15..cb75221 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -340,45 +340,34 @@ DEALINGS IN THE SOFTWARE. click (https://github.com/pallets/click) -Copyright © 2014 by the Pallets team. +Copyright 2014 Pallets -Some rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -Redistribution and use in source and binary forms of the software as -well as documentation, with or without modification, are permitted -provided that the following conditions are met: - -- Redistributions of source code must retain the above copyright +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -- Redistributions in binary form must reproduce the above copyright +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -- Neither the name of the copyright holder nor the names of its +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - ----- - -Click uses parts of optparse written by Gregory P. Ward and maintained -by the Python Software Foundation. This is limited to code in parser.py. - -Copyright © 2001-2006 Gregory P. Ward. All rights reserved. -Copyright © 2002-2006 Python Software Foundation. All rights reserved. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- @@ -957,6 +946,11 @@ ruamel.yaml (https://bitbucket.org/ruamel/yaml) singularity (https://github.com/sylabs/singularity) +Copyright (c) 2015-2017, Gregory M. Kurtzer. All rights reserved. +Copyright (c) 2016-2017, The Regents of the University of California. All right reserved. +Copyright (c) 2017, SingularityWare, LLC. All rights reserved. +Copyright (c) 2018-2019, Sylabs, Inc. All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/docker/libtorrent/1.0.x/Dockerfile b/docker/libtorrent/1.0.x/Dockerfile deleted file mode 100644 index d26ea46..0000000 --- a/docker/libtorrent/1.0.x/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Dockerfile for Azure/batch-shipyard (Cascade libtorrent multi-stage base) - -FROM alpine:3.7 -MAINTAINER Fred Park - -# set environment variables -ENV libtorrent_version=1.0.11 \ - CFLAGS=-lstdc++ PYTHON=/usr/bin/python3 PYTHON_VERSION=3.6 - -# build libtorrent-rasterbar for python3 and cleanup packaging -RUN apk update \ - && apk add --update --no-cache \ - musl build-base python3 python3-dev libressl-dev libffi-dev \ - boost boost-dev boost-python3 file curl tar bash \ - && curl -SL https://github.com/arvidn/libtorrent/releases/download/libtorrent-${libtorrent_version//./_}/libtorrent-rasterbar-${libtorrent_version}.tar.gz -o libtorrent-${libtorrent_version}.tar.gz \ - && tar zxvpf libtorrent-${libtorrent_version}.tar.gz \ - && cd libtorrent-rasterbar-${libtorrent_version} \ - && ./configure --prefix=/usr --enable-debug=no --enable-python-binding --with-boost-system=boost_system \ - && make -j"$(nproc)" install \ - && ldconfig /usr/lib \ - && cd .. \ - && rm -rf libtorrent-rasterbar-${libtorrent_version} \ - && rm -f libtorrent-${libtorrent_version}.tar.gz \ - && apk del --purge \ - build-base python3-dev libressl-dev libffi-dev python boost-dev \ - file curl \ - && rm /var/cache/apk/* diff --git a/docker/libtorrent/1.1.x/Dockerfile b/docker/libtorrent/1.1.x/Dockerfile deleted file mode 100644 index 1b1a342..0000000 --- a/docker/libtorrent/1.1.x/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -# Dockerfile for Azure/batch-shipyard (Cascade libtorrent multi-stage base) - -FROM alpine:3.8 -MAINTAINER Fred Park - -# set environment variables -ENV libtorrent_version=1.1.9 \ - CFLAGS=-lstdc++ PYTHON=/usr/bin/python3 PYTHON_VERSION=3.6 - -# build libtorrent-rasterbar for python3 and cleanup packaging -RUN apk update \ - && apk add --update --no-cache \ - musl build-base python3 python3-dev libressl-dev libffi-dev \ - boost boost-dev boost-python3 file curl tar bash \ - && curl -SL https://github.com/arvidn/libtorrent/releases/download/libtorrent-${libtorrent_version//./_}/libtorrent-rasterbar-${libtorrent_version}.tar.gz -o libtorrent-${libtorrent_version}.tar.gz \ - && tar zxvpf libtorrent-${libtorrent_version}.tar.gz \ - && cd libtorrent-rasterbar-${libtorrent_version} \ - && ./configure --prefix=/usr --enable-debug=no --enable-python-binding --with-boost-system=boost_system \ - && make -j"$(nproc)" install \ - && ldconfig /usr/lib \ - && cd .. \ - && rm -rf libtorrent-rasterbar-${libtorrent_version} \ - && rm -f libtorrent-${libtorrent_version}.tar.gz \ - && apk del --purge \ - build-base python3-dev libressl-dev libffi-dev python boost-dev \ - file curl \ - && rm /var/cache/apk/* diff --git a/docker/linux/cli/Dockerfile b/images/docker/linux/cli/Dockerfile similarity index 96% rename from docker/linux/cli/Dockerfile rename to images/docker/linux/cli/Dockerfile index d8e66b8..743c43e 100644 --- a/docker/linux/cli/Dockerfile +++ b/images/docker/linux/cli/Dockerfile @@ -13,7 +13,7 @@ RUN apk update \ && git clone -b $GIT_BRANCH --single-branch https://github.com/Azure/batch-shipyard.git /opt/batch-shipyard \ && cd /opt/batch-shipyard \ && git checkout $GIT_COMMIT \ - && rm -rf .git .github \ + && rm -rf .git .github .vsts \ && rm -f .git* .travis.yml *.yml install* \ && python3 -m pip install --no-cache-dir --upgrade pip \ && pip3 install --no-cache-dir -r requirements.txt \ diff --git a/docker/linux/cli/hooks/build b/images/docker/linux/cli/hooks/build similarity index 100% rename from docker/linux/cli/hooks/build rename to images/docker/linux/cli/hooks/build diff --git a/docker/singularity/Dockerfile b/images/docker/singularity/Dockerfile similarity index 100% rename from docker/singularity/Dockerfile rename to images/docker/singularity/Dockerfile diff --git a/docker/singularity/build-images.sh b/images/docker/singularity/build-images.sh similarity index 100% rename from docker/singularity/build-images.sh rename to images/docker/singularity/build-images.sh diff --git a/docker/slurm/centos/7/Dockerfile b/images/docker/slurm/centos/7/Dockerfile similarity index 100% rename from docker/slurm/centos/7/Dockerfile rename to images/docker/slurm/centos/7/Dockerfile diff --git a/docker/slurm/centos/7/slurmctld.service b/images/docker/slurm/centos/7/slurmctld.service similarity index 100% rename from docker/slurm/centos/7/slurmctld.service rename to images/docker/slurm/centos/7/slurmctld.service diff --git a/docker/slurm/centos/7/slurmd.service b/images/docker/slurm/centos/7/slurmd.service similarity index 100% rename from docker/slurm/centos/7/slurmd.service rename to images/docker/slurm/centos/7/slurmd.service diff --git a/docker/slurm/centos/7/slurmdbd.service b/images/docker/slurm/centos/7/slurmdbd.service similarity index 100% rename from docker/slurm/centos/7/slurmdbd.service rename to images/docker/slurm/centos/7/slurmdbd.service diff --git a/docker/slurm/ubuntu/16.04/Dockerfile b/images/docker/slurm/ubuntu/16.04/Dockerfile similarity index 100% rename from docker/slurm/ubuntu/16.04/Dockerfile rename to images/docker/slurm/ubuntu/16.04/Dockerfile diff --git a/docker/slurm/ubuntu/16.04/slurmctld.service b/images/docker/slurm/ubuntu/16.04/slurmctld.service similarity index 100% rename from docker/slurm/ubuntu/16.04/slurmctld.service rename to images/docker/slurm/ubuntu/16.04/slurmctld.service diff --git a/docker/slurm/ubuntu/16.04/slurmd.service b/images/docker/slurm/ubuntu/16.04/slurmd.service similarity index 100% rename from docker/slurm/ubuntu/16.04/slurmd.service rename to images/docker/slurm/ubuntu/16.04/slurmd.service diff --git a/docker/slurm/ubuntu/16.04/slurmdbd.service b/images/docker/slurm/ubuntu/16.04/slurmdbd.service similarity index 100% rename from docker/slurm/ubuntu/16.04/slurmdbd.service rename to images/docker/slurm/ubuntu/16.04/slurmdbd.service diff --git a/docker/slurm/ubuntu/18.04/Dockerfile b/images/docker/slurm/ubuntu/18.04/Dockerfile similarity index 100% rename from docker/slurm/ubuntu/18.04/Dockerfile rename to images/docker/slurm/ubuntu/18.04/Dockerfile diff --git a/docker/slurm/ubuntu/18.04/slurmctld.service b/images/docker/slurm/ubuntu/18.04/slurmctld.service similarity index 100% rename from docker/slurm/ubuntu/18.04/slurmctld.service rename to images/docker/slurm/ubuntu/18.04/slurmctld.service diff --git a/docker/slurm/ubuntu/18.04/slurmd.service b/images/docker/slurm/ubuntu/18.04/slurmd.service similarity index 100% rename from docker/slurm/ubuntu/18.04/slurmd.service rename to images/docker/slurm/ubuntu/18.04/slurmd.service diff --git a/docker/slurm/ubuntu/18.04/slurmdbd.service b/images/docker/slurm/ubuntu/18.04/slurmdbd.service similarity index 100% rename from docker/slurm/ubuntu/18.04/slurmdbd.service rename to images/docker/slurm/ubuntu/18.04/slurmdbd.service diff --git a/docker/windows/azure.ico b/images/docker/windows/azure.ico similarity index 100% rename from docker/windows/azure.ico rename to images/docker/windows/azure.ico diff --git a/docker/windows/cargo/Dockerfile b/images/docker/windows/cargo/Dockerfile similarity index 100% rename from docker/windows/cargo/Dockerfile rename to images/docker/windows/cargo/Dockerfile diff --git a/docker/windows/cli/Dockerfile b/images/docker/windows/cli/Dockerfile similarity index 98% rename from docker/windows/cli/Dockerfile rename to images/docker/windows/cli/Dockerfile index 1393bac..824b6f2 100644 --- a/docker/windows/cli/Dockerfile +++ b/images/docker/windows/cli/Dockerfile @@ -19,6 +19,7 @@ RUN git clone -b $Env:GIT_BRANCH --single-branch https://github.com/Azure/batch- pip install --no-cache-dir --no-deps -r req_nodeps.txt ; \ Remove-Item .git -Force -Recurse ; \ Remove-Item .git* -Force -Recurse ; \ + Remove-Item .vsts -Force -Recurse ; \ Remove-Item .travis.yml -Force -Recurse ; \ Remove-Item *.yml -Force -Recurse ; \ Remove-Item install* -Force -Recurse diff --git a/docker/windows/file_version_info.txt b/images/docker/windows/file_version_info.txt similarity index 100% rename from docker/windows/file_version_info.txt rename to images/docker/windows/file_version_info.txt diff --git a/docker/gen_3rd_party_notices.sh b/images/gen_3rd_party_notices.sh similarity index 100% rename from docker/gen_3rd_party_notices.sh rename to images/gen_3rd_party_notices.sh diff --git a/images/singularity/cli.def b/images/singularity/cli.def new file mode 100644 index 0000000..4949b75 --- /dev/null +++ b/images/singularity/cli.def @@ -0,0 +1,22 @@ +# Singularity for Azure/batch-shipyard (cli) + +Bootstrap: library +From: alpine:3.9 + +%post +apk update +apk add --update --no-cache \ + musl build-base python3 python3-dev openssl-dev libffi-dev \ + ca-certificates openssl openssh-client rsync git bash +git clone -b master --single-branch --depth 5 https://github.com/Azure/batch-shipyard.git /opt/batch-shipyard +cd /opt/batch-shipyard +rm -rf .git .github .vsts +rm -f .git* .travis.yml *.yml install* +pip3 install --no-cache-dir -r requirements.txt +pip3 install --no-cache-dir --no-deps -r req_nodeps.txt +python3 -m compileall -f /opt/batch-shipyard/shipyard.py /opt/batch-shipyard/convoy +apk del --purge build-base python3-dev openssl-dev libffi-dev git +rm /var/cache/apk/* + +%runscript +exec /opt/batch-shipyard/shipyard.py "$@" diff --git a/recipes/HPCG-Infiniband-IntelMPI/README.md b/recipes/HPCG-Infiniband-IntelMPI/README.md index db45ac7..a459d3e 100644 --- a/recipes/HPCG-Infiniband-IntelMPI/README.md +++ b/recipes/HPCG-Infiniband-IntelMPI/README.md @@ -37,8 +37,8 @@ included in the Linpack image. The global configuration should set the following properties: * `singularity_images` array must have a reference to a valid HPCG image that can be run with Intel MPI and Infiniband. This can be -`shub://alfpark/linpack` which is -published on [Singularity Hub](https://www.singularity-hub.org/containers/496). +`library://alfpark/linpack/linpack:2018-intel-mkl` which is +published on the [Singularity Library](https://cloud.sylabs.io/library/alfpark/linpack/linpack). ### Jobs Configuration #### Docker-based diff --git a/recipes/HPLinpack-Infiniband-IntelMPI/README.md b/recipes/HPLinpack-Infiniband-IntelMPI/README.md index 368f082..bb4f5d4 100644 --- a/recipes/HPLinpack-Infiniband-IntelMPI/README.md +++ b/recipes/HPLinpack-Infiniband-IntelMPI/README.md @@ -36,8 +36,8 @@ published on [Docker Hub](https://hub.docker.com/r/alfpark/linpack). The global configuration should set the following properties: * `singularity_images` array must have a reference to a valid HPLinpack image that can be run with Intel MPI and Infiniband. This can be -`shub://alfpark/linpack` which is -published on [Singularity Hub](https://www.singularity-hub.org/containers/496). +`library://alfpark/linpack/linpack:2018-intel-mkl` which is +published on the [Singularity Library](https://cloud.sylabs.io/library/alfpark/linpack/linpack). ### Jobs Configuration #### Docker-based