Fix nvidia-docker overwriting daemon.json

- Update packer scripts

contrib/packer/centos-7.3-GPU/bootstrap.sh

@@ -22,18 +22,6 @@ yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce
 yum makecache -y fast
 yum install -y docker-ce-${dockerversion}
 
-# prep docker
-systemctl stop docker.service
-rm -rf /var/lib/docker
-mkdir -p /etc/docker
-echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
-sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
-systemctl daemon-reload
-# do not auto-enable docker to start due to temp disk issues
-systemctl disable docker.service
-systemctl start docker.service
-systemctl status docker.service
-
 # install nvidia driver/nvidia-docker
 set +e
 out=$(lspci)
@@ -72,3 +60,17 @@ EOF
     nvidia-docker version
 fi
 set -e
+
+# prep docker
+systemctl stop docker.service
+rm -rf /var/lib/docker
+mkdir -p /etc/docker
+echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json.merge
+python -c "import json;a=json.load(open('/etc/docker/daemon.json.merge'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
+rm -f /etc/docker/daemon.json.merge
+sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
+systemctl daemon-reload
+# do not auto-enable docker to start due to temp disk issues
+systemctl disable docker.service
+systemctl start docker.service
+systemctl status docker.service

contrib/packer/ubuntu-16.04-GPU+IB/bootstrap.sh

@@ -22,18 +22,6 @@ apt-get update
 apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \
     docker-ce=$dockerversion
 
-# prep docker
-systemctl stop docker.service
-rm -rf /var/lib/docker
-mkdir -p /etc/docker
-echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
-sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
-systemctl daemon-reload
-# do not auto-enable docker to start due to temp disk issues
-systemctl disable docker.service
-systemctl start docker.service
-systemctl status docker.service
-
 # install nvidia driver/nvidia-docker
 set +e
 out=$(lspci)
@@ -73,6 +61,20 @@ EOF
 fi
 set -e
 
+# prep docker
+systemctl stop docker.service
+rm -rf /var/lib/docker
+mkdir -p /etc/docker
+echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"fd://\", \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json.merge
+python -c "import json;a=json.load(open('/etc/docker/daemon.json.merge'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
+rm -f /etc/docker/daemon.json.merge
+sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
+systemctl daemon-reload
+# do not auto-enable docker to start due to temp disk issues
+systemctl disable docker.service
+systemctl start docker.service
+systemctl status docker.service
+
 # install userland IB requirements
 apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \
     libdapl2 libmlx4-1

contrib/packer/ubuntu-16.04-GPU/bootstrap.sh

@@ -20,18 +20,6 @@ apt-get update
 apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommends \
     docker-ce=$dockerversion
 
-# prep docker
-systemctl stop docker.service
-rm -rf /var/lib/docker
-mkdir -p /etc/docker
-echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
-sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
-systemctl daemon-reload
-# do not auto-enable docker to start due to temp disk issues
-systemctl disable docker.service
-systemctl start docker.service
-systemctl status docker.service
-
 # install nvidia driver/nvidia-docker
 set +e
 out=$(lspci)
@@ -70,3 +58,17 @@ EOF
     nvidia-docker version
 fi
 set -e
+
+# prep docker
+systemctl stop docker.service
+rm -rf /var/lib/docker
+mkdir -p /etc/docker
+echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"fd://\", \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json.merge
+python -c "import json;a=json.load(open('/etc/docker/daemon.json.merge'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
+rm -f /etc/docker/daemon.json.merge
+sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
+systemctl daemon-reload
+# do not auto-enable docker to start due to temp disk issues
+systemctl disable docker.service
+systemctl start docker.service
+systemctl status docker.service

contrib/packer/ubuntu-16.04-IB/bootstrap.sh

@@ -24,7 +24,7 @@ apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommen
 systemctl stop docker.service
 rm -rf /var/lib/docker
 mkdir -p /etc/docker
-echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
+echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"fd://\", \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
 sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
 systemctl daemon-reload
 # do not auto-enable docker to start due to temp disk issues

contrib/packer/ubuntu-16.04/bootstrap.sh

@@ -22,7 +22,7 @@ apt-get install -y -q -o Dpkg::Options::="--force-confnew" --no-install-recommen
 systemctl stop docker.service
 rm -rf /var/lib/docker
 mkdir -p /etc/docker
-echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
+echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"fd://\", \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json
 sed -i 's|^ExecStart=/usr/bin/dockerd.*|ExecStart=/usr/bin/dockerd|' /lib/systemd/system/docker.service
 systemctl daemon-reload
 # do not auto-enable docker to start due to temp disk issues

convoy/batch.py

@@ -333,6 +333,8 @@ def _reboot_node(batch_client, pool_id, node_id, wait):
     :param str node_id: node id to delete
     :param bool wait: wait for node to enter rebooting state
     """
+    if util.is_none_or_empty(node_id):
+        raise ValueError('node id must be specified for reboot')
     logger.info('Rebooting node {} in pool {}'.format(node_id, pool_id))
     batch_client.compute_node.reboot(
         pool_id=pool_id,

scripts/shipyard_nodeprep.sh

@@ -6,6 +6,9 @@ set -o pipefail
 # consts
 MOUNTS_PATH=$AZ_BATCH_NODE_ROOT_DIR/mounts
 
+# mutables
+USER_MOUNTPOINT=
+
 # globals
 azureblob=0
 azurefile=0
@@ -243,6 +246,15 @@ EOF
     fi
     refresh_package_index $offer
     install_packages $offer nvidia-docker2
+    # we need to merge daemon configs
+    if [ $offer == "ubuntuserver" ]; then
+        python -c "import json;a=json.load(open('/etc/docker/daemon.json.dpkg-old'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
+        rm -f /etc/docker/daemon.json.dpkg-old
+    elif [[ $offer == centos* ]]; then
+        echo "{ \"graph\": \"$USER_MOUNTPOINT/docker\", \"hosts\": [ \"unix:///var/run/docker.sock\", \"tcp://127.0.0.1:2375\" ] }" > /etc/docker/daemon.json.merge
+        python -c "import json;a=json.load(open('/etc/docker/daemon.json.merge'));b=json.load(open('/etc/docker/daemon.json'));a.update(b);f=open('/etc/docker/daemon.json','w');json.dump(a,f);f.close();"
+        rm -f /etc/docker/daemon.json.merge
+    fi
     pkill -SIGHUP dockerd
     nvidia-docker version
 }