Azure
/
aztk
зеркало из https://github.com/Azure/aztk.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
aztk/docker-image/build.sh

144 строки
5.9 KiB
Bash
Исходник Обычный вид История

Feature: refactor docker images (#510) * add spark2.3.0 hadoop2.8.3 dockerfile * start update to docker image * add SPARK_DIST_CLASSPATH to bashrc, source .bashrc in docker run * add maven install for jars * docker image update and code fix * add libthrift (still broken) * start image refactor, build from source, * add refactor to r base image * finish refactor r image * add storage jars and deps * exclude netty to get rid of dependency conflict * add miniconda image * update 2.2.0 base, anaconda image * remove unused cuda-8.0 image * start pipenv implementation * miniconda version arg * update anaconda and miniconda image * style * pivot to virtualenv * remove virtualenv from path when submitting apps * flatten layers * explicit calls to aztk python instead of activating virtualenv * update base, miniconda, anaconda * add compatibility version for base aztk images * typo fix * update pom * update environment variable name * update environment variables * add anaconda images base & gpu * update gpu and miniconda base images * create venv in cluster create * update base docker files, remove virtualenv * fix path * add exclusion to base images * update r images * delete python images (in favor of anaconda and miniconda) * add miniconda gpu images * update comment * update aztk_version_compatibility to dokcer image version * add a build script * virutalenv->pipenv, add pipfile & pipfile.lock remove secretstorage * aztk/staging->aztk/spark * remove jars, add .null to keep directory * update pipfile, update jupyter and jupyterlab * update default images * update base images to fix hdfs * update build script with correct path * add spark1.6.3 anaconda, miniconda, r base and gpu images * update build script to include spark1.6.3 * mkdir out * exclude commons lang and slf4j dependencies * mkdir out * no fail if dir exists * update node_scripts * update env var name * update env var name * fix the docker_repo docs * master->0.7.0
2018-05-01 03:19:01 +03:00
#/bin/bash
# setup docker to build on /mnt instead of /var/lib/docker
echo '{
"graph": "/mnt",
"storage-driver": "overlay"
}' > /etc/docker/daemon.json
service docker restart
mkdir -p out
# base 1.6.3
docker build base/spark1.6.3/ --tag aztk/spark:v0.1.0-spark1.6.3-base > out/base-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-base
# base 2.1.0
docker build base/spark2.1.0/ --tag aztk/spark:v0.1.0-spark2.1.0-base > out/base-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-base
# base 2.2.0
docker build base/spark2.2.0/ --tag aztk/spark:v0.1.0-spark2.2.0-base > out/base-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-base
# base 2.3.0
docker build base/spark2.3.0/ --tag aztk/spark:v0.1.0-spark2.3.0-base > out/base-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-base
# miniconda-base 1.6.3
docker build miniconda/spark1.6.3/base/ --tag aztk/spark:v0.1.0-spark1.6.3-miniconda-base > out/miniconda-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-miniconda-base
# miniconda-base 2.1.0
docker build miniconda/spark2.1.0/base/ --tag aztk/spark:v0.1.0-spark2.1.0-miniconda-base > out/miniconda-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-miniconda-base
# miniconda-base 2.2.0
docker build miniconda/spark2.2.0/base --tag aztk/spark:v0.1.0-spark2.2.0-miniconda-base > out/miniconda-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-miniconda-base
# miniconda-base 2.3.0
docker build miniconda/spark2.3.0/base/ --tag aztk/spark:v0.1.0-spark2.3.0-miniconda-base > out/miniconda-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-miniconda-base
# anaconda-base 1.6.3
docker build anaconda/spark1.6.3/base/ --tag aztk/spark:v0.1.0-spark1.6.3-anaconda-base > out/anaconda-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-anaconda-base
# anaconda-base 2.1.0
docker build anaconda/spark2.1.0/base/ --tag aztk/spark:v0.1.0-spark2.1.0-anaconda-base > out/anaconda-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-anaconda-base
# anaconda-base 2.2.0
docker build anaconda/spark2.2.0/base/ --tag aztk/spark:v0.1.0-spark2.2.0-anaconda-base > out/anaconda-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-anaconda-base
# anaconda-base 2.3.0
docker build anaconda/spark2.3.0/base/ --tag aztk/spark:v0.1.0-spark2.3.0-anaconda-base > out/anaconda-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-anaconda-base
# r-base 1.6.3
docker build r/spark1.6.3/base/ --tag aztk/spark:v0.1.0-spark1.6.3-r-base > out/r-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-r-base
# r-base 2.1.0
docker build r/spark2.1.0/base/ --tag aztk/spark:v0.1.0-spark2.1.0-r-base > out/r-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-r-base
# r-base 2.2.0
docker build r/spark2.2.0/base/ --tag aztk/spark:v0.1.0-spark2.2.0-r-base > out/r-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-r-base
# r-base 2.3.0
docker build r/spark2.3.0/base/ --tag aztk/spark:v0.1.0-spark2.3.0-r-base > out/r-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-r-base
##################
# GPU #
##################
# gpu 1.6.3
docker build gpu/spark1.6.3/ --tag aztk/spark:v0.1.0-spark1.6.3-gpu > out/gpu-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-gpu
# gpu 2.1.0
docker build gpu/spark2.1.0/ --tag aztk/spark:v0.1.0-spark2.1.0-gpu > out/gpu-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-gpu
# gpu 2.2.0
docker build gpu/spark2.2.0/ --tag aztk/spark:v0.1.0-spark2.2.0-gpu > out/gpu-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-gpu
# gpu 2.3.0
docker build gpu/spark2.3.0/ --tag aztk/spark:v0.1.0-spark2.3.0-gpu > out/gpu-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-gpu
# miniconda-gpu 1.6.3
docker build miniconda/spark1.6.3/gpu/ --tag aztk/spark:v0.1.0-spark1.6.3-miniconda-gpu > out/miniconda-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-miniconda-gpu
# miniconda-gpu 2.1.0
docker build miniconda/spark2.1.0/gpu/ --tag aztk/spark:v0.1.0-spark2.1.0-miniconda-gpu > out/miniconda-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-miniconda-gpu
# miniconda-gpu 2.2.0
docker build miniconda/spark2.2.0/gpu --tag aztk/spark:v0.1.0-spark2.2.0-miniconda-gpu > out/miniconda-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-miniconda-gpu
# miniconda-gpu 2.3.0
docker build miniconda/spark2.3.0/gpu/ --tag aztk/spark:v0.1.0-spark2.3.0-miniconda-gpu > out/miniconda-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-miniconda-gpu
# anaconda-gpu 1.6.3
docker build anaconda/spark1.6.3/gpu/ --tag aztk/spark:v0.1.0-spark1.6.3-anaconda-gpu > out/anaconda-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-anaconda-gpu
# anaconda-gpu 2.1.0
docker build anaconda/spark2.1.0/gpu/ --tag aztk/spark:v0.1.0-spark2.1.0-anaconda-gpu > out/anaconda-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-anaconda-gpu
# anaconda-gpu 2.2.0
docker build anaconda/spark2.2.0/gpu/ --tag aztk/spark:v0.1.0-spark2.2.0-anaconda-gpu > out/anaconda-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-anaconda-gpu
# anaconda-gpu 2.3.0
docker build anaconda/spark2.3.0/gpu/ --tag aztk/spark:v0.1.0-spark2.3.0-anaconda-gpu > out/anaconda-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-anaconda-gpu
# r-gpu 1.6.3
docker build r/spark1.6.3/gpu/ --tag aztk/spark:v0.1.0-spark1.6.3-r-gpu > out/r-spark1.6.3.out &&
docker push aztk/spark:v0.1.0-spark1.6.3-r-gpu
# r-gpu 2.1.0
docker build r/spark2.1.0/gpu/ --tag aztk/spark:v0.1.0-spark2.1.0-r-gpu > out/r-spark2.1.0.out &&
docker push aztk/spark:v0.1.0-spark2.1.0-r-gpu
# r-gpu 2.2.0
docker build r/spark2.2.0/gpu/ --tag aztk/spark:v0.1.0-spark2.2.0-r-gpu > out/r-spark2.2.0.out &&
docker push aztk/spark:v0.1.0-spark2.2.0-r-gpu
# r-gpu 2.3.0
docker build r/spark2.3.0/gpu/ --tag aztk/spark:v0.1.0-spark2.3.0-r-gpu > out/r-spark2.3.0.out &&
docker push aztk/spark:v0.1.0-spark2.3.0-r-gpu