Bug 1363444 - Vagrant: Update to Ubuntu 16.04

At some point soon we'll want to switch Heroku stage/production to
their new Heroku-16 (Ubuntu 16.04) stack, so it makes sense to trial
it locally first. Their Heroku-16 docker image will make switching
to docker simpler too.

Notable changes to the provision steps:
* We're using the Bento boxes rather than the 'official' Canonical
  ones, at the recommendation of Vagrant:
  https://www.vagrantup.com/docs/boxes.html#official-boxes
* openjdk-8-jre-headless is now available from the ubuntu.com
  repository, so doesn't require a PPA.
* There isn't an official mysql-server-5.6 package (only 5.7+), so
  we have to start using a PPA.
* Services must now be managed using systemd rather than SysV init.
* The `eth0` network interface has changed name:
  https://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
* The `vendor-libmysqlclient.sh` script will still be run on Ubuntu
  14.04 on Travis/Heroku for now, so has to be compatible with both.
* The previous temporary cleanup steps can be removed since the
  vagrant destroy means a clean slate.

Vagrantfile

@@ -21,7 +21,10 @@ Vagrant.configure("2") do |config|
   config.vm.synced_folder ".", "/home/vagrant/treeherder", type: "nfs"
 
   config.vm.provider "virtualbox" do |vb, override|
-    override.vm.box = "ubuntu/trusty64"
+    # The Bento boxes (https://github.com/chef/bento) are recommended over the
+    # Canonical ones, since they more closely follow Vagrant best practices.
+    override.vm.box = "bento/ubuntu-16.04"
+    override.vm.box_version = ">= 2.3.5"
     vb.name = "treeherder"
     vb.memory = "3072"
   end

bin/vendor-libmysqlclient.sh

@@ -29,20 +29,21 @@
 set -euo pipefail
 
 VENDOR_DIR="$1"
-VERSION="5.7.18"
+UBUNTU_VERSION="$(lsb_release -r -s)"
+MYSQL_VERSION="5.7.18"
 PACKAGE_URLS=(
     # We have to use packages from mysql.com since there is no Ubuntu distro
     # release available for MySQL 5.7 on Ubuntu 14.04.
-    "https://dev.mysql.com/get/Downloads/MySQL-5.7/libmysqlclient20_${VERSION}-1ubuntu14.04_amd64.deb"
-    "https://dev.mysql.com/get/Downloads/MySQL-5.7/libmysqlclient-dev_${VERSION}-1ubuntu14.04_amd64.deb"
+    "https://dev.mysql.com/get/Downloads/MySQL-5.7/libmysqlclient20_${MYSQL_VERSION}-1ubuntu${UBUNTU_VERSION}_amd64.deb"
+    "https://dev.mysql.com/get/Downloads/MySQL-5.7/libmysqlclient-dev_${MYSQL_VERSION}-1ubuntu${UBUNTU_VERSION}_amd64.deb"
 )
 
 # Skip vendoring if libmysqlclient-dev's `mysql_config` exists and reports the correct version.
-if [[ "$(mysql_config --version 2>&1)" == "$VERSION" ]]; then
+if [[ "$(mysql_config --version 2>&1)" == "$MYSQL_VERSION" ]]; then
     exit 0
 fi
 
-echo "-----> Vendoring libmysqlclient $VERSION."
+echo "-----> Vendoring libmysqlclient $MYSQL_VERSION."
 
 # We manually extract the packages rather than using apt-get install, since:
 #  - On Heroku we don't have sudo, so need to vendor the package somewhere other

vagrant/.profile

@@ -10,7 +10,7 @@
 # As such, it's just easier to re-run the commands on each login since they take <30ms.
 sudo sysctl -q -w net.ipv4.conf.all.route_localnet=1
 sudo iptables -t nat --flush
-sudo iptables -t nat -A PREROUTING -i eth0 -p tcp -j DNAT --to 127.0.0.1
+sudo iptables -t nat -A PREROUTING -i enp0s3 -p tcp -j DNAT --to 127.0.0.1
 
 PS1='\[\e[0;31m\]\u\[\e[m\] \[\e[1;34m\]\w\[\e[m\] \$ '
 echo "Type 'thelp' to see a list of Treeherder-specific helper aliases"
@@ -48,12 +48,12 @@ function thqueuespurge {
 
 function threstartmemcached {
     echo "Restarting memcache"
-    sudo service memcached restart
+    sudo systemctl restart memcached.service
 }
 
 function threstartrabbitmq {
     echo "Restarting rabbitmq"
-    sudo service rabbitmq-server restart
+    sudo systemctl restart rabbitmq-server.service
 }
 
 function thresetall {

vagrant/setup.sh

@@ -4,6 +4,12 @@
 # Make non-zero exit codes & other errors fatal.
 set -euo pipefail
 
+if [[ "$(lsb_release -r -s)" != "16.04" ]]; then
+    echo "This machine needs to be switched to the new Ubuntu 16.04 image."
+    echo "Please run 'vagrant destroy -f && vagrant up' from the host."
+    exit 1
+fi
+
 SRC_DIR="$HOME/treeherder"
 PYTHON_DIR="$HOME/python"
 
@@ -23,15 +29,16 @@ ln -sf "$SRC_DIR/vagrant/.profile" "$HOME/.profile"
 sudo ln -sf "$SRC_DIR/vagrant/env.sh" /etc/profile.d/treeherder.sh
 . /etc/profile.d/treeherder.sh
 
-if [[ ! -f /etc/apt/sources.list.d/openjdk-r-ppa-trusty.list ]]; then
-    echo '-----> Adding APT repository for OpenJDK'
-    sudo add-apt-repository -y ppa:openjdk-r/ppa 2>&1
+if [[ ! -f /etc/apt/sources.list.d/ondrej-ubuntu-mysql-5_6-xenial.list ]]; then
+    echo '-----> Adding APT repository for MySQL 5.6'
+    # Neither Ubuntu nor Oracle have an official Xenial mysql-5.6 package.
+    sudo add-apt-repository -y ppa:ondrej/mysql-5.6 2>&1
 fi
 
 if [[ ! -f /etc/apt/sources.list.d/nodesource.list ]]; then
     echo '-----> Adding APT repository for Node.js'
     curl -sSf https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
-    echo 'deb https://deb.nodesource.com/node_7.x trusty main' | sudo tee /etc/apt/sources.list.d/nodesource.list > /dev/null
+    echo 'deb https://deb.nodesource.com/node_7.x xenial main' | sudo tee /etc/apt/sources.list.d/nodesource.list > /dev/null
 fi
 
 if [[ ! -f /etc/apt/sources.list.d/yarn.list ]]; then
@@ -44,7 +51,7 @@ echo '-----> Installing/updating APT packages'
 sudo -E apt-get -yqq update
 # g++ is required by Brotli
 # libmemcached-dev and zlib1g-dev are required by pylibmc
-# openjdk-7-jre-headless is required by Elasticsearch
+# openjdk-8-jre-headless is required by Elasticsearch
 sudo -E apt-get -yqq install --no-install-recommends \
     g++ \
     git \
@@ -61,19 +68,16 @@ if [[ "$(dpkg-query --show --showformat='${Version}' elasticsearch 2>&1)" != "$E
     echo '-----> Installing Elasticsearch'
     curl -sSfo /tmp/elasticsearch.deb "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb"
     sudo dpkg -i /tmp/elasticsearch.deb
-    sudo update-rc.d elasticsearch defaults 95 10
-    # Clean up old JDK version, otherwise Elasticsearch won't know which to use,
-    # and existing Vagrant VMs will break.
-    sudo apt-get remove -y openjdk-7-jre-headless
     # Override the new ES 5.x default minimum heap size of 2GB.
     sudo sed -i 's/.*ES_JAVA_OPTS=.*/ES_JAVA_OPTS="-Xms256m -Xmx1g"/' /etc/default/elasticsearch
-    sudo service elasticsearch restart
+    sudo systemctl enable elasticsearch.service 2>&1
+    sudo systemctl restart elasticsearch.service
 fi
 
 if ! cmp -s vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf; then
     echo '-----> Configuring MySQL'
     sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
-    sudo service mysql restart
+    sudo systemctl restart mysql.service
 fi
 
 if [[ "$($PYTHON_DIR/bin/python --version 2>&1)" != *"$PYTHON_VERSION" ]]; then
@@ -81,7 +85,9 @@ if [[ "$($PYTHON_DIR/bin/python --version 2>&1)" != *"$PYTHON_VERSION" ]]; then
     rm -rf "$PYTHON_DIR"
     mkdir -p "$PYTHON_DIR"
     # Uses the Heroku Python buildpack's binaries for parity with production.
-    curl -sSf "https://lang-python.s3.amazonaws.com/cedar-14/runtimes/python-$PYTHON_VERSION.tar.gz" | tar -xz -C "$PYTHON_DIR"
+    curl -sSf "https://lang-python.s3.amazonaws.com/heroku-16/runtimes/python-$PYTHON_VERSION.tar.gz" | tar -xz -C "$PYTHON_DIR"
+    # Work around https://github.com/heroku/heroku-buildpack-python/issues/379
+    rm -rf "$PYTHON_DIR/lib/python2.7/site-packages/"*
 fi
 
 if [[ "$($PYTHON_DIR/bin/pip --version 2>&1)" != *"$PIP_VERSION"* ]]; then
@@ -114,8 +120,5 @@ echo '-----> Running Django migrations and loading reference data'
 echo '-----> Performing cleanup'
 # Celery sometimes gets stuck and requires that celerybeat-schedule be deleted.
 rm -f celerybeat-schedule || true
-# TODO: Remove in a few weeks.
-sudo apt-get remove -y varnish
-rm -rf "$HOME/venv"
 
 echo '-----> Setup complete!'