Bug 1340554 - Update from Elasticsearch 2.x to 5.x

* ES 5.x now requires JDK 8, and Ubuntu 14.04 only ships with openjdk 7,
  so a third party PPA must be used in Vagrant:
  https://launchpad.net/~openjdk-r/+archive/ubuntu/ppa
* ES 5.x changed the way it manages the heap size, such that:
  - The variables for controlling it have changed (now set via eg
    `ES_JAVA_OPTS="-Xms1g -Xmx1g"` or in the jvm.options file). See:
    https://www.elastic.co/guide/en/elasticsearch/reference/5.2/heap-size.html
  - The default heap size has increased from ~(min:256MB, max:1GB) to
    (min: 2GB, max: 2GB) which causes OOM in the VM, unless either
    lowered back down or the VM RAM increased.
* The Python ES clients must be updated to the latest releases:
  https://elasticsearch-py.readthedocs.io/en/master/#compatibility
* The previous test failures were fixed in #2403.
* The Vagrant provision now also waits for Elasticsearch to be ready
  before trying to run the Django migrations, since Elasticsearch can
  take a while to start (and always has). This prevents failures when
  the pip/yarn install steps are no-ops (when already up to date),
  causing the Django migration to run immediately after the ES install
  step.

.travis.yml

@@ -74,7 +74,7 @@ matrix:
       services:
         - rabbitmq
       before_install:
-        - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb
+        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb
         - sudo service elasticsearch start
         - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
         - sudo service mysql restart
@@ -89,7 +89,7 @@ matrix:
       install:
         - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
       before_script:
-        - while ! curl localhost:9200 &>/dev/null; do sleep 1; done
+        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
       script:
         # Several security features in settings.py (eg setting HSTS headers) are conditional on
         # 'https://' being in the site URL. In addition, we override the test environment's debug
@@ -109,7 +109,7 @@ matrix:
       services:
         - rabbitmq
       before_install:
-        - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb
+        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb
         - sudo service elasticsearch start
         - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
         - sudo service mysql restart
@@ -122,7 +122,7 @@ matrix:
       install:
         - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
       before_script:
-        - while ! curl localhost:9200 &>/dev/null; do sleep 1; done
+        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
       script:
         - py.test tests/e2e/ tests/etl/ tests/log_parser/ --runslow
 
@@ -138,7 +138,7 @@ matrix:
       services:
         - rabbitmq
       before_install:
-        - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb
+        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb
         - sudo service elasticsearch start
         - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
         - sudo service mysql restart
@@ -151,7 +151,7 @@ matrix:
       install:
         - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
       before_script:
-        - while ! curl localhost:9200 &>/dev/null; do sleep 1; done
+        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
       script:
         - py.test tests/webapp/ --runslow
 
@@ -169,7 +169,7 @@ matrix:
       services:
         - rabbitmq
       before_install:
-        - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb
+        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb
         - sudo service elasticsearch start
         - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
         - sudo service mysql restart
@@ -191,7 +191,7 @@ matrix:
         - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
         - yarn install --frozen-lockfile --no-bin-links
       before_script:
-        - while ! curl localhost:9200 &>/dev/null; do sleep 1; done
+        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
         - "export DISPLAY=:99.0"
         - "sh -e /etc/init.d/xvfb start"
         - sleep 3 # give xvfb some time to start

requirements/common.txt

@@ -127,13 +127,8 @@ pylibmc==1.5.2 \
 
 django-pylibmc==0.6.1 --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93
 
-elasticsearch==2.4.1 \
-    --hash=sha256:bb8f9a365ba6650d599428538c8aed42033264661d8f7d353da59d5892305f72 \
-    --hash=sha256:fead47ebfcaabd1c53dbfc21403eb99ac207eef76de8002fe11a1c8ec9589ce2  # pyup: >=2,<3 # Client has to match ES server version.
-
-elasticsearch-dsl==2.2.0 \
-    --hash=sha256:99bbb4dcbcfb5db4f57499237f24acf1397543e895e99994a09af2a6fbef93bc \
-    --hash=sha256:c8132c6e1bdfc5c345999d3949dd53a53b59ea73e2ee005f695de78be0552ceb  # pyup: >=2,<3 # Client has to match ES server version.
+elasticsearch==5.3.0 --hash=sha256:aea16d6d6ad03a509c9f591eee9ce4ce62918d8ba26bf6513097223929be0a09
+elasticsearch-dsl==5.2.0 --hash=sha256:1a816fed5a0b9cbc862670f3455aa151e75f6042db38a1b008f73cc12173e0d0
 
 # required by elasticsearch
 urllib3==1.20 --hash=sha256:b64c0faa183e9e9e76193146c4147e82a734982c6b6719dca851d6cc4ec90c01

vagrant/setup.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 SRC_DIR="$HOME/treeherder"
 VENV_DIR="$HOME/venv"
-ELASTICSEARCH_VERSION="2.4.4"
+ELASTICSEARCH_VERSION="5.3.1"
 
 export PATH="$VENV_DIR/bin:$PATH"
 # Suppress prompts during apt-get invocations.
@@ -26,6 +26,11 @@ if [[ ! -f /etc/apt/sources.list.d/fkrull-deadsnakes-python2_7-trusty.list ]]; t
     sudo add-apt-repository -y ppa:fkrull/deadsnakes-python2.7 2>&1
 fi
 
+if [[ ! -f /etc/apt/sources.list.d/openjdk-r-ppa-trusty.list ]]; then
+    echo '-----> Adding APT repository for OpenJDK'
+    sudo add-apt-repository -y ppa:openjdk-r/ppa 2>&1
+fi
+
 if [[ ! -f /etc/apt/sources.list.d/nodesource.list ]]; then
     echo '-----> Adding APT repository for Node.js'
     curl -sSf https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add -
@@ -51,7 +56,7 @@ sudo -E apt-get -yqq install --no-install-recommends \
     memcached \
     mysql-server-5.6 \
     nodejs \
-    openjdk-7-jre-headless \
+    openjdk-8-jre-headless \
     python2.7 \
     python2.7-dev \
     rabbitmq-server \
@@ -61,10 +66,15 @@ sudo -E apt-get -yqq install --no-install-recommends \
 
 if [[ "$(dpkg-query --show --showformat='${Version}' elasticsearch 2>&1)" != "$ELASTICSEARCH_VERSION" ]]; then
     echo '-----> Installing Elasticsearch'
-    curl -sSfo /tmp/elasticsearch.deb "https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ELASTICSEARCH_VERSION/elasticsearch-$ELASTICSEARCH_VERSION.deb"
+    curl -sSfo /tmp/elasticsearch.deb "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb"
     sudo dpkg -i /tmp/elasticsearch.deb
     sudo update-rc.d elasticsearch defaults 95 10
-    sudo service elasticsearch start
+    # Clean up old JDK version, otherwise Elasticsearch won't know which to use,
+    # and existing Vagrant VMs will break.
+    sudo apt-get remove -y openjdk-7-jre-headless
+    # Override the new ES 5.x default minimum heap size of 2GB.
+    sudo sed -i 's/.*ES_JAVA_OPTS=.*/ES_JAVA_OPTS="-Xms256m -Xmx1g"/' /etc/default/elasticsearch
+    sudo service elasticsearch restart
 fi
 
 if ! cmp -s vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf; then
@@ -110,6 +120,9 @@ echo '-----> Initialising MySQL database'
 mysql -u root -e 'GRANT ALL PRIVILEGES ON *.* to root@"%"'
 mysql -u root -e 'CREATE DATABASE IF NOT EXISTS treeherder'
 
+echo '-----> Waiting for Elasticsearch to be ready'
+while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
+
 echo '-----> Running Django migrations and loading reference data'
 ./manage.py migrate --noinput
 ./manage.py load_initial_data