From 7a4595d44f9ba9ffb0da6338157151dc6ee80170 Mon Sep 17 00:00:00 2001 From: Fred Park Date: Mon, 2 Dec 2019 16:46:59 +0000 Subject: [PATCH] Add Ubuntu 18.04 Packer Script for SR-IOV IB/RDMA --- .../ubuntu-18.04/sriov_rdma/bootstrap.sh | 235 ++++++++++++++++++ .../packer/ubuntu-18.04/sriov_rdma/build.json | 64 +++++ 2 files changed, 299 insertions(+) create mode 100644 contrib/packer/ubuntu-18.04/sriov_rdma/bootstrap.sh create mode 100644 contrib/packer/ubuntu-18.04/sriov_rdma/build.json diff --git a/contrib/packer/ubuntu-18.04/sriov_rdma/bootstrap.sh b/contrib/packer/ubuntu-18.04/sriov_rdma/bootstrap.sh new file mode 100644 index 0000000..86fedc3 --- /dev/null +++ b/contrib/packer/ubuntu-18.04/sriov_rdma/bootstrap.sh @@ -0,0 +1,235 @@ +#!/usr/bin/env bash + +set -e +set -o pipefail + +dockerversion=$1 +shift + +OS=ubuntu18.04 +OS_CODENAME=bionic +USER_MOUNTPOINT=/mnt + +# install docker +apt-get update +apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \ + apt-transport-https ca-certificates curl gnupg2 software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \ + docker-ce="5:${dockerversion}~3-0~ubuntu-${OS_CODENAME}" \ + docker-ce-cli="5:${dockerversion}~3-0~ubuntu-${OS_CODENAME}" + +# prep docker +systemctl stop docker.service +rm -rf /var/lib/docker +mkdir -p /etc/docker +echo "{ \"data-root\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json +sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service +systemctl daemon-reload +# do not auto-enable docker to start due to temp disk issues +systemctl disable docker.service +systemctl start docker.service +systemctl status --no-pager docker.service + +# install pre-reqs +apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \ + build-essential libnuma-dev binutils binutils-dev zlib1g zlib1g-dev environment-modules dkms + +# path setup +INSTALL_PREFIX=/opt +MODULEFILES_PATH=/usr/share/modules/modulefiles/mpi +mkdir -p $MODULEFILES_PATH + +# install Mellanox OFED +OFED_VERSION=4.7-1.0.0.1 +pushd /tmp +curl -fSsL http://content.mellanox.com/ofed/MLNX_OFED-${OFED_VERSION}/MLNX_OFED_LINUX-${OFED_VERSION}-${OS}-x86_64.tgz | tar -zxpf - +cd MLNX_OFED_LINUX* +./mlnxofedinstall --force --without-fw-update -vv +popd + +# install HPC-X +HPCX_VERSION=v2.5.0 +pushd ${INSTALL_PREFIX} +curl -fSsL http://content.mellanox.com/hpc/hpc-x/v2.5/hpcx-${HPCX_VERSION}-gcc-MLNX_OFED_LINUX-${OFED_VERSION}-${OS}-x86_64.tbz | tar -jxpf - +HPCX_PATH=${INSTALL_PREFIX}/hpcx-${HPCX_VERSION}-gcc-MLNX_OFED_LINUX-${OFED_VERSION}-${OS}-x86_64 +ls ${HPCX_PATH} +HCOLL_PATH=${HPCX_PATH}/hcoll +ls ${HCOLL_PATH} +UCX_PATH=${HPCX_PATH}/ucx +ls ${UCX_PATH} +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/hpcx-${HPCX_VERSION} +#%Module 1.0 +# +# HPCx ${HPCX_VERSION} +# +conflict mpi +module load ${INSTALL_PREFIX}/hpcx-${HPCX_VERSION}-gcc-MLNX_OFED_LINUX-${OFED_VERSION}-${OS}-x86_64/modulefiles/hpcx +EOF +ln -s ${MODULEFILES_PATH}/hpcx-${HPCX_VERSION} ${MODULEFILES_PATH}/hpcx + +# Add sharp lib to ld library path +SHARP_LIB_PATH=/opt/mellanox/sharp/lib +echo "$SHARP_LIB_PATH" > /etc/ld.so.conf.d/mellanox.conf +ldconfig + +# install MPICH +MPICH_VERSION=3.3.2 +pushd /tmp +curl -fSsL http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar -zxpf - +cd mpich-${MPICH_VERSION} +# need to add sharp lib so it can find HCOLL +LD_LIBRARY_PATH=${HCOLL_PATH}/lib:${SHARP_LIB_PATH}:${LD_LIBRARY_PATH} ./configure --prefix=${INSTALL_PREFIX}/mpich-${MPICH_VERSION} --with-ucx=${UCX_PATH} --with-hcoll=${HCOLL_PATH} --enable-g=none --enable-fast=yes --with-device=ch4:ucx +make +make install +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/mpich-${MPICH_VERSION} +#%Module 1.0 +# +# MPICH ${MPICH_VERSION} +# +conflict mpi +prepend-path PATH ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/bin +prepend-path LD_LIBRARY_PATH ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/lib +prepend-path MANPATH ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/share/man +setenv MPI_BIN ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/bin +setenv MPI_INCLUDE ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/include +setenv MPI_LIB ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/lib +setenv MPI_MAN ${INSTALL_PREFIX}/mpich-${MPICH_VERSION}/share/man +setenv MPI_HOME ${INSTALL_PREFIX}/mpich-${MPICH_VERSION} +EOF +ln -s ${MODULEFILES_PATH}/mpich-${MPICH_VERSION} ${MODULEFILES_PATH}/mpich + +# install OpenMPI +OMPI_VERSION=4.0.2 +pushd /tmp +curl -fSsL https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-${OMPI_VERSION}.tar.gz | tar -zxpf - +cd openmpi-${OMPI_VERSION} +# need to add sharp lib so it can find HCOLL +LD_LIBRARY_PATH=${HCOLL_PATH}/lib:${SHARP_LIB_PATH}:${LD_LIBRARY_PATH} ./configure --prefix=${INSTALL_PREFIX}/openmpi-${OMPI_VERSION} --with-ucx=${UCX_PATH} --with-hcoll=${HCOLL_PATH} --enable-mpirun-prefix-by-default --with-platform=contrib/platform/mellanox/optimized +make -j +make install +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/openmpi-${OMPI_VERSION} +#%Module 1.0 +# +# OpenMPI ${OMPI_VERSION} +# +conflict mpi +prepend-path PATH ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/bin +prepend-path LD_LIBRARY_PATH ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/lib +prepend-path MANPATH ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/share/man +setenv MPI_BIN ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/bin +setenv MPI_INCLUDE ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/include +setenv MPI_LIB ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/lib +setenv MPI_MAN ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION}/share/man +setenv MPI_HOME ${INSTALL_PREFIX}/openmpi-${OMPI_VERSION} +EOF +ln -s ${MODULEFILES_PATH}/openmpi-${OMPI_VERSION} ${MODULEFILES_PATH}/openmpi + +# install MVAPICH +MVAPICH_VERSION=2.3.2 +pushd /tmp +curl -fSsL http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-${MVAPICH_VERSION}.tar.gz | tar -zxpf - +cd mvapich2-${MVAPICH_VERSION} +./configure --prefix=${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION} --enable-g=none --enable-fast=yes +make -j +make install +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/mvapich2-${MVAPICH_VERSION} +#%Module 1.0 +# +# MVAPICH2 ${MVAPICH_VERSION} +# +conflict mpi +prepend-path PATH ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/bin +prepend-path LD_LIBRARY_PATH ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/lib +prepend-path MANPATH ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/share/man +setenv MPI_BIN ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/bin +setenv MPI_INCLUDE ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/include +setenv MPI_LIB ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/lib +setenv MPI_MAN ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION}/share/man +setenv MPI_HOME ${INSTALL_PREFIX}/mvapich2-${MVAPICH_VERSION} +EOF +ln -s ${MODULEFILES_PATH}/mvapich2-${MVAPICH_VERSION} ${MODULEFILES_PATH}/mvapich2 + +# install intel mpi 2018 runtime +IMPI_2018_VERSION=l_mpi_2018.5.288 +pushd /tmp +curl -fSsL http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15614/${IMPI_2018_VERSION}.tgz | tar -zxpf - +cd ./${IMPI_2018_VERSION} +sed -i -e 's/^ACCEPT_EULA=.*/ACCEPT_EULA=accept/g' silent.cfg +sed -i -e 's,^ARCH_SELECTED=.*,ARCH_SELECTED=INTEL64,g' silent.cfg +./install.sh --silent silent.cfg +# enable ptrace for non-root non-debugger processes for intel mpi +echo 0 | tee /proc/sys/kernel/yama/ptrace_scope +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/impi_${IMPI_2018_VERSION} +#%Module 1.0 +# +# Intel MPI ${IMPI_2018_VERSION} +# +conflict mpi +prepend-path PATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64/bin +prepend-path LD_LIBRARY_PATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64/lib +prepend-path MANPATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/man +setenv MPI_BIN ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64/bin +setenv MPI_INCLUDE ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64/include +setenv MPI_LIB ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64/lib +setenv MPI_MAN ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/man +setenv MPI_HOME ${INSTALL_PREFIX}/intel/impi/${IMPI_2018_VERSION}/intel64 +EOF +ln -s ${MODULEFILES_PATH}/impi_${IMPI_2018_VERSION} ${MODULEFILES_PATH}/impi + +# install intel mpi 2019 runtime +IMPI_2019_VERSION=l_mpi_2019.5.281 +pushd /tmp +curl -fSsL http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15838/${IMPI_2019_VERSION}.tgz | tar -zxpf - +cd ./${IMPI_2019_VERSION} +sed -i -e 's/^ACCEPT_EULA=.*/ACCEPT_EULA=accept/g' silent.cfg +sed -i -e 's,^ARCH_SELECTED=.*,ARCH_SELECTED=INTEL64,g' silent.cfg +./install.sh --silent silent.cfg +popd +# create module file +cat << EOF >> ${MODULEFILES_PATH}/impi_${IMPI_2019_VERSION} +#%Module 1.0 +# +# Intel MPI ${IMPI_2019_VERSION} +# +conflict mpi +prepend-path PATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64/bin +prepend-path LD_LIBRARY_PATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64/lib +prepend-path MANPATH ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/man +setenv MPI_BIN ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64/bin +setenv MPI_INCLUDE ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64/include +setenv MPI_LIB ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64/lib +setenv MPI_MAN ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/man +setenv MPI_HOME ${INSTALL_PREFIX}/intel/impi/${IMPI_2019_VERSION}/intel64 +EOF +ln -s ${MODULEFILES_PATH}/impi_${IMPI_2019_VERSION} ${MODULEFILES_PATH}/impi-2019 + +# enable RDMA in agent conf +sed -i 's/^# OS.EnableRDMA=.*/OS.EnableRDMA=y/g' /etc/waagent.conf +sed -i 's/^# AutoUpdate.Enabled=.*/AutoUpdate.Enabled=y/g' /etc/waagent.conf + +# adjust limits +echo "" >> /etc/security/limits.conf +echo "* hard memlock unlimited" >> /etc/security/limits.conf +echo "* soft memlock unlimited" >> /etc/security/limits.conf +echo "* hard nofile 65535" >> /etc/security/limits.conf +echo "* soft nofile 65535" >> /etc/security/limits.conf + +# enable zone reclaim mode +echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf +sysctl -p + +# cleanup apt +apt autoremove -y diff --git a/contrib/packer/ubuntu-18.04/sriov_rdma/build.json b/contrib/packer/ubuntu-18.04/sriov_rdma/build.json new file mode 100644 index 0000000..f674eb2 --- /dev/null +++ b/contrib/packer/ubuntu-18.04/sriov_rdma/build.json @@ -0,0 +1,64 @@ +{ + "variables": { + "image_name": "", + "image_version": "", + "image_resource_group": "", + "sig_name": "", + "sig_replication_regions": "", + "sig_resource_group": "", + "subscription_id": "", + "directory_id": "", + "application_id": "", + "auth_key": "=", + "location": "", + "vm_size": "STANDARD_D4_V2", + "docker_version": "19.03.5", + "bootstrap": "bootstrap.sh" + }, + "builders": [ + { + "type": "azure-arm", + + "subscription_id": "{{user `subscription_id`}}", + "tenant_id": "{{user `directory_id`}}", + "client_id": "{{user `application_id`}}", + "client_secret": "{{user `auth_key`}}", + + "os_type": "Linux", + "image_publisher": "Canonical", + "image_offer": "UbuntuServer", + "image_sku": "18.04-LTS", + + "location": "{{user `location`}}", + "vm_size": "{{user `vm_size`}}", + + "managed_image_name": "{{user `image_name`}}", + "managed_image_resource_group_name": "{{user `image_resource_group`}}", + + "shared_image_gallery_destination": { + "resource_group": "{{user `sig_resource_group`}}", + "gallery_name": "{{user `sig_name`}}", + "image_name": "{{user `image_name`}}", + "image_version": "{{user `image_version`}}", + "replication_regions": "{{user `sig_replication_regions`}}" + } + } + ], + "provisioners": [ + { + "type": "file", + "source": "{{user `bootstrap`}}", + "destination": "/tmp/{{user `bootstrap`}}" + }, + { + "execute_command": "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'", + "inline": [ + "chmod +x /tmp/{{user `bootstrap`}}", + "/tmp/{{user `bootstrap`}} {{user `docker_version`}}", + "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync" + ], + "inline_shebang": "/bin/bash -e", + "type": "shell" + } + ] +}