treeherder/.travis.yml

dist: trusty
# The fully visualised "sudo" GCE environments are faster for longer running jobs.
sudo: required
# Use the latest Travis images since they are more up to date than the stable release.
group: edge
env:
  global:
    - BROKER_URL='amqp://guest:guest@localhost:5672//'
    - DATABASE_URL='mysql://root@localhost/test_treeherder'
    - ELASTICSEARCH_URL='http://127.0.0.1:9200'
    - TREEHERDER_DJANGO_SECRET_KEY='secretkey-of-at-50-characters-to-pass-check-deploy'
matrix:
  include:

    # Job 1: Linters
    - env: python2-linters
      sudo: false
      language: python
      python: "2.7.14"
      cache:
        directories:
          - ~/venv
      before_install:
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
      script:
        - pip check
        - python lints/queuelint.py
        - flake8 --show-source
        - isort --check-only --diff --quiet

    # Job 2: Linters
    - env: python3-linters
      sudo: false
      language: python
      python: "3.6.2"
      cache:
        directories:
          - ~/venv
      before_install:
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
      script:
        - pip check

    # Job 3: Nodejs UI tests
    - env: ui-tests
      language: node_js
      node_js: "8.7.0"
      cache:
        # Note: This won't re-use the same cache as the linters job,
        # since caches are tied to the language/version combination.
        directories:
          - node_modules
      addons:
        firefox: latest
      before_install:
        # Use newer yarn than that pre-installed in the Travis image.
        - curl -sSfL https://yarnpkg.com/latest.tar.gz | tar -xz --strip-components=1 -C "$HOME"
      install:
        # `--frozen-lockfile` will catch cases where people have forgotten to update `yarn.lock`.
        # `--no-bin-links` is only necessary on Windows hosts, but we include here to ensure
        # that the package.json scripts aren't relying on symlinks that won't exist elsewhere.
        - yarn install --frozen-lockfile --no-bin-links
      before_script:
        # Required for Karma tests (http://docs.travis-ci.com/user/gui-and-headless-browsers/)
        - export DISPLAY=:99.0
        - sh -e /etc/init.d/xvfb start
      script:
        - yarn test
        - yarn build

    # Job 4: Python Tests Chunk A
    - env: python-tests-main
      language: python
      python: "2.7.14"
      cache:
        directories:
          - ~/venv
      services:
        - rabbitmq
      before_install:
        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.deb && sudo dpkg -i --force-confold ~/elasticsearch.deb
        - sudo service elasticsearch restart
        - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
        # Using tmpfs to improve MySQL performance reduces pytest runtime by 30%.
        - sudo mkdir /mnt/ramdisk
        - sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk
        - sudo stop mysql
        - sudo mv /var/lib/mysql /mnt/ramdisk
        - sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql
        - echo 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7' | sudo tee /etc/apt/sources.list.d/mysql.list > /dev/null
        - sudo -E apt-get -yqq update
        - sudo -E apt-get -yqq install --no-install-recommends --allow-unauthenticated mysql-server libmysqlclient-dev
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
        # Create the test database for `manage.py check --deploy`.
        - mysql -u root -e 'create database test_treeherder;'
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
      before_script:
        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
      script:
        # Several security features in settings.py (eg setting HSTS headers) are conditional on
        # 'https://' being in the site URL. In addition, we override the test environment's debug
        # value so the tests pass. The real environment variable will be checked during deployment.
        - SITE_URL='https://treeherder.dev' TREEHERDER_DEBUG='False' ./manage.py check --deploy --fail-level WARNING
        - pytest tests/ --runslow --ignore=tests/etl/ --ignore=tests/log_parser/ --ignore=tests/webapp/api/ --ignore=tests/selenium/ --ignore=tests/jenkins/

    # Job 5: Python Tests Chunk B
    - env: python-tests-etl-logparser
      language: python
      python: "2.7.14"
      cache:
        directories:
          - ~/venv
      services:
        - rabbitmq
      before_install:
        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.deb && sudo dpkg -i --force-confold ~/elasticsearch.deb
        - sudo service elasticsearch restart
        - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
        # Using tmpfs to improve MySQL performance reduces pytest runtime by 30%.
        - sudo mkdir /mnt/ramdisk
        - sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk
        - sudo stop mysql
        - sudo mv /var/lib/mysql /mnt/ramdisk
        - sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql
        - echo 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7' | sudo tee /etc/apt/sources.list.d/mysql.list > /dev/null
        - sudo -E apt-get -yqq update
        - sudo -E apt-get -yqq install --no-install-recommends --allow-unauthenticated mysql-server libmysqlclient-dev
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
      before_script:
        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
      script:
        - pytest tests/etl/ tests/log_parser/ --runslow

    # Job 6: Python Tests Chunk C
    - env: python-tests-rest-api
      language: python
      python: "2.7.14"
      cache:
        directories:
          - ~/venv
      services:
        - rabbitmq
      before_install:
        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.deb && sudo dpkg -i --force-confold ~/elasticsearch.deb
        - sudo service elasticsearch restart
        - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
        # Using tmpfs to improve MySQL performance reduces pytest runtime by 30%.
        - sudo mkdir /mnt/ramdisk
        - sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk
        - sudo stop mysql
        - sudo mv /var/lib/mysql /mnt/ramdisk
        - sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql
        - echo 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7' | sudo tee /etc/apt/sources.list.d/mysql.list > /dev/null
        - sudo -E apt-get -yqq update
        - sudo -E apt-get -yqq install --no-install-recommends --allow-unauthenticated mysql-server libmysqlclient-dev
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
      before_script:
        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
      script:
        - pytest tests/webapp/api/ --runslow

    # Job 7: Python Tests - Selenium integration
    - env: python-tests-selenium
      language: python
      python: "2.7.14"
      cache:
        directories:
          - ~/venv
          - node_modules
      addons:
        firefox: latest
      services:
        - rabbitmq
      before_install:
        - curl -sSo ~/elasticsearch.deb https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.0.deb && sudo dpkg -i --force-confold ~/elasticsearch.deb
        - sudo service elasticsearch restart
        - sudo cp vagrant/mysql.cnf /etc/mysql/conf.d/treeherder.cnf
        # Using tmpfs to improve MySQL performance reduces pytest runtime by 30%.
        - sudo mkdir /mnt/ramdisk
        - sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk
        - sudo stop mysql
        - sudo mv /var/lib/mysql /mnt/ramdisk
        - sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql
        - echo 'deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7' | sudo tee /etc/apt/sources.list.d/mysql.list > /dev/null
        - sudo -E apt-get -yqq update
        - sudo -E apt-get -yqq install --no-install-recommends --allow-unauthenticated mysql-server libmysqlclient-dev
        # Create a clean virtualenv rather than using the one given to us,
        # to work around: https://github.com/travis-ci/travis-ci/issues/4873
        - if [[ ! -f ~/venv/bin/activate ]]; then virtualenv -p python ~/venv; fi
        - source ~/venv/bin/activate
        - curl -sSfL https://github.com/mozilla/geckodriver/releases/download/v0.19.0/geckodriver-v0.19.0-linux64.tar.gz | tar -zxC $HOME/bin
        - nvm install 8.7.0
        # Use newer yarn than that pre-installed in the Travis image.
        - curl -sSfL https://yarnpkg.com/latest.tar.gz | tar -xz --strip-components=1 -C "$HOME"
      install:
        - pip install --disable-pip-version-check --require-hashes -r requirements/common.txt -r requirements/dev.txt
        - yarn install --frozen-lockfile --no-bin-links
      before_script:
        - while ! curl "$ELASTICSEARCH_URL" &> /dev/null; do sleep 1; done
        - "export DISPLAY=:99.0"
        - "sh -e /etc/init.d/xvfb start"
      script:
        - yarn build
        - py.test tests/selenium/ --runselenium --driver Firefox

notifications:
  email:
    on_success: never
on_failure: always