From 18e8971532962f54d2ad5b9d98e2b42d5fdeb0df Mon Sep 17 00:00:00 2001 From: Ed Morley Date: Wed, 19 Apr 2017 13:44:46 +0100 Subject: [PATCH] Bug 1340554 - Update from Elasticsearch 2.x to 5.x * ES 5.x now requires JDK 8, and Ubuntu 14.04 only ships with openjdk 7, so a third party PPA must be used in Vagrant: https://launchpad.net/~openjdk-r/+archive/ubuntu/ppa * ES 5.x changed the way it manages the heap size, such that: - The variables for controlling it have changed (now set via eg `ES_JAVA_OPTS="-Xms1g -Xmx1g"` or in the jvm.options file). See: https://www.elastic.co/guide/en/elasticsearch/reference/5.2/heap-size.html - The default heap size has increased from ~(min:256MB, max:1GB) to (min: 2GB, max: 2GB) which causes OOM in the VM, unless either lowered back down or the VM RAM increased. * The Python ES clients must be updated to the latest releases: https://elasticsearch-py.readthedocs.io/en/master/#compatibility * The previous test failures were fixed in #2403. * The Vagrant provision now also waits for Elasticsearch to be ready before trying to run the Django migrations, since Elasticsearch can take a while to start (and always has). This prevents failures when the pip/yarn install steps are no-ops (when already up to date), causing the Django migration to run immediately after the ES install step. --- .travis.yml | 16 ++++++++-------- requirements/common.txt | 9 ++------- vagrant/setup.sh | 21 +++++++++++++++++---- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a92c2806..d9db56144 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,7 @@ matrix: services: - rabbitmq before_install: - - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb + - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb - sudo service elasticsearch start - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf - sudo service mysql restart @@ -89,7 +89,7 @@ matrix: install: - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt before_script: - - while ! curl localhost:9200 &>/dev/null; do sleep 1; done + - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done script: # Several security features in settings.py (eg setting HSTS headers) are conditional on # 'https://' being in the site URL. In addition, we override the test environment's debug @@ -109,7 +109,7 @@ matrix: services: - rabbitmq before_install: - - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb + - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb - sudo service elasticsearch start - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf - sudo service mysql restart @@ -122,7 +122,7 @@ matrix: install: - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt before_script: - - while ! curl localhost:9200 &>/dev/null; do sleep 1; done + - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done script: - py.test tests/e2e/ tests/etl/ tests/log_parser/ --runslow @@ -138,7 +138,7 @@ matrix: services: - rabbitmq before_install: - - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb + - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb - sudo service elasticsearch start - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf - sudo service mysql restart @@ -151,7 +151,7 @@ matrix: install: - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt before_script: - - while ! curl localhost:9200 &>/dev/null; do sleep 1; done + - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done script: - py.test tests/webapp/ --runslow @@ -169,7 +169,7 @@ matrix: services: - rabbitmq before_install: - - curl -sSo ~/elasticsearch.deb https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.4/elasticsearch-2.4.4.deb && sudo dpkg -i ~/elasticsearch.deb + - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.1.deb && sudo dpkg -i ~/elasticsearch.deb - sudo service elasticsearch start - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf - sudo service mysql restart @@ -191,7 +191,7 @@ matrix: - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt - yarn install --frozen-lockfile --no-bin-links before_script: - - while ! curl localhost:9200 &>/dev/null; do sleep 1; done + - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" - sleep 3 # give xvfb some time to start diff --git a/requirements/common.txt b/requirements/common.txt index 2e8c1bda5..866765396 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -127,13 +127,8 @@ pylibmc==1.5.2 \ django-pylibmc==0.6.1 --hash=sha256:9cffdee703aaf9ebc029d9dbdee8abdd0723564b95e4b2ac59e4a668b8e58f93 -elasticsearch==2.4.1 \ - --hash=sha256:bb8f9a365ba6650d599428538c8aed42033264661d8f7d353da59d5892305f72 \ - --hash=sha256:fead47ebfcaabd1c53dbfc21403eb99ac207eef76de8002fe11a1c8ec9589ce2 # pyup: >=2,<3 # Client has to match ES server version. - -elasticsearch-dsl==2.2.0 \ - --hash=sha256:99bbb4dcbcfb5db4f57499237f24acf1397543e895e99994a09af2a6fbef93bc \ - --hash=sha256:c8132c6e1bdfc5c345999d3949dd53a53b59ea73e2ee005f695de78be0552ceb # pyup: >=2,<3 # Client has to match ES server version. +elasticsearch==5.3.0 --hash=sha256:aea16d6d6ad03a509c9f591eee9ce4ce62918d8ba26bf6513097223929be0a09 +elasticsearch-dsl==5.2.0 --hash=sha256:1a816fed5a0b9cbc862670f3455aa151e75f6042db38a1b008f73cc12173e0d0 # required by elasticsearch urllib3==1.20 --hash=sha256:b64c0faa183e9e9e76193146c4147e82a734982c6b6719dca851d6cc4ec90c01 diff --git a/vagrant/setup.sh b/vagrant/setup.sh index ccfab72cf..f1a9329dd 100644 --- a/vagrant/setup.sh +++ b/vagrant/setup.sh @@ -6,7 +6,7 @@ set -euo pipefail SRC_DIR="$HOME/treeherder" VENV_DIR="$HOME/venv" -ELASTICSEARCH_VERSION="2.4.4" +ELASTICSEARCH_VERSION="5.3.1" export PATH="$VENV_DIR/bin:$PATH" # Suppress prompts during apt-get invocations. @@ -26,6 +26,11 @@ if [[ ! -f /etc/apt/sources.list.d/fkrull-deadsnakes-python2_7-trusty.list ]]; t sudo add-apt-repository -y ppa:fkrull/deadsnakes-python2.7 2>&1 fi +if [[ ! -f /etc/apt/sources.list.d/openjdk-r-ppa-trusty.list ]]; then + echo '-----> Adding APT repository for OpenJDK' + sudo add-apt-repository -y ppa:openjdk-r/ppa 2>&1 +fi + if [[ ! -f /etc/apt/sources.list.d/nodesource.list ]]; then echo '-----> Adding APT repository for Node.js' curl -sSf https://deb.nodesource.com/gpgkey/nodesource.gpg.key | sudo apt-key add - @@ -51,7 +56,7 @@ sudo -E apt-get -yqq install --no-install-recommends \ memcached \ mysql-server-5.6 \ nodejs \ - openjdk-7-jre-headless \ + openjdk-8-jre-headless \ python2.7 \ python2.7-dev \ rabbitmq-server \ @@ -61,10 +66,15 @@ sudo -E apt-get -yqq install --no-install-recommends \ if [[ "$(dpkg-query --show --showformat='${Version}' elasticsearch 2>&1)" != "$ELASTICSEARCH_VERSION" ]]; then echo '-----> Installing Elasticsearch' - curl -sSfo /tmp/elasticsearch.deb "https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ELASTICSEARCH_VERSION/elasticsearch-$ELASTICSEARCH_VERSION.deb" + curl -sSfo /tmp/elasticsearch.deb "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb" sudo dpkg -i /tmp/elasticsearch.deb sudo update-rc.d elasticsearch defaults 95 10 - sudo service elasticsearch start + # Clean up old JDK version, otherwise Elasticsearch won't know which to use, + # and existing Vagrant VMs will break. + sudo apt-get remove -y openjdk-7-jre-headless + # Override the new ES 5.x default minimum heap size of 2GB. + sudo sed -i 's/.*ES_JAVA_OPTS=.*/ES_JAVA_OPTS="-Xms256m -Xmx1g"/' /etc/default/elasticsearch + sudo service elasticsearch restart fi if ! cmp -s vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf; then @@ -110,6 +120,9 @@ echo '-----> Initialising MySQL database' mysql -u root -e 'GRANT ALL PRIVILEGES ON *.* to root@"%"' mysql -u root -e 'CREATE DATABASE IF NOT EXISTS treeherder' +echo '-----> Waiting for Elasticsearch to be ready' +while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done + echo '-----> Running Django migrations and loading reference data' ./manage.py migrate --noinput ./manage.py load_initial_data