Improve Travis and Tox configuration (#6732)

Some changes in here:
* Making tox a bit more the centralized point for configurying job execution
* Move Heroku code paths into tox from .travis.yml
* Move to more modern Travis image (xenial 16.04 -> bionic 18.04)
* Make Python tests outside of Docker to run slow tests (DB required)
* Cache tox packages to speed up following Travis runs
* Remove coverage data before gathering new data (to prevent using old data)

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
 # Use the latest Travis images since they are more up to date than the stable release.
 group: edge
 jobs:
@@ -24,42 +24,32 @@ jobs:
         - yarn codecov
 
     # Run Heroku build & release related code
-    # This build is configured to catch issues on PRs that would only be detected as part of Heroku's
-    # build step when the code has already been merged to master
-    # The step ./bin/post_compile requires the output of `yarn build`, thus, we need to build
-    # both the JS and Python builds
     - env:
         - NODE_ENV=production YARN_PRODUCTION=true
-      # Since we're using Python/pip and Node/yarn use the generic image
-      language: generic
+      language: python
+      python: '3.7'
       cache:
         directories:
           - $HOME/.cache/yarn
           - $HOME/.cache/pip
           - node_modules
-      before_install:
-        # XXX: I have not been able to install 3.7.2 to match runtime.txt
-        - pyenv global 3.7.1
-        - nvm install 12.14.1
-        - nvm use 12.14.1
-        # Steps to validate versions in use
-        - python --version
-        - pip --version
-        - node --version
-        - yarn --version
+          - .tox/py37
       install:
-        - pip install -r requirements.txt
-        - yarn install
+        - pip install tox
       script:
-        - yarn heroku-postbuild
-        - ./manage.py collectstatic --noinput
-        # This generates the revision and does the Brotly/Gzip compression
-        - ./bin/post_compile
+        - tox -e heroku
 
     # Run Python tests & linters outside of the Docker containers
     - language: python
+      services:
+        - docker
       python: '3.7'
-      cache: pip
+      cache:
+        directories:
+          - $HOME/.cache/yarn
+          - $HOME/.cache/pip
+          - node_modules
+          - .tox/py37
       install:
         - pip install tox
       script:
@@ -70,23 +60,32 @@ jobs:
     - language: shell
       services:
         - docker
+      cache:
+        directories:
+          - $HOME/.cache/pip
+          - .tox/py37
       install:
-        - pip install tox --user
-        - pip install codecov --user
+        - pip install tox
+        - pip install codecov
       script:
         - tox -e docker
-        - codecov -f coverage.xml
+        # It will fail if there's no coverage data
+        - codecov --required -f coverage.xml
 
     # Run Python Selenium tests
     - language: node_js
       services:
         - docker
       # The Node version here must be kept in sync with that in `package.json`.
-      node_js: '12.13.0'
-      cache: yarn
+      node_js: '12.14.1'
+      cache:
+        directories:
+          - $HOME/.cache/pip
+          - $HOME/.cache/yarn
+          - .tox/py37
       install:
-        - pip install tox --user
-        - pip install codecov --user
+        - pip install tox
+        - pip install codecov
       script:
         - tox -e selenium
         - codecov -f coverage.xml

package.json

@@ -92,7 +92,7 @@
   "scripts": {
     "build": "node ./node_modules/webpack/bin/webpack.js --mode production",
     "build:dev": "node ./node_modules/webpack/bin/webpack.js --mode development",
-    "codecov": "node ./node_modules/codecov/bin/codecov && rm -rf coverage",
+    "codecov": "rm -rf coverage && node ./node_modules/codecov/bin/codecov",
     "format": "node ./node_modules/prettier/bin-prettier.js --write \"**/*.{css,html,js,jsx,json,md,yaml,yml}\"",
     "format:check": "node ./node_modules/prettier/bin-prettier.js --check \"**/*.{css,html,js,jsx,json,md,yaml,yml}\"",
     "heroku-postbuild": "yarn build",

tox.ini

@@ -13,7 +13,6 @@ commands_pre =
     docker-compose up --detach mysql redis rabbitmq
     pip install -r {toxinidir}/requirements/dev.txt
     pip install -r {toxinidir}/requirements/common.txt
-    {toxinidir}/manage.py check
 commands =
     pip check
     {toxinidir}/lints/queuelint.py
@@ -21,9 +20,10 @@ commands =
     black --check .
     shellcheck initialize_data.sh
     shellcheck docker/entrypoint.sh
+    {toxinidir}/manage.py check
     sh -c "SITE_URL=https://treeherder.dev TREEHERDER_DEBUG=False ./manage.py check --deploy --fail-level WARNING"
-    # Exercising running tests outside of Docker
-    pytest tests/ --ignore=tests/selenium
+    # Running slow tests (DB required)
+    pytest tests/ --runslow --ignore=tests/selenium
 commands_post =
     # This is to deal with running the containers with --detached
     docker-compose down
@@ -36,10 +36,28 @@ commands =
     mkdocs build
 commands_post =
 
+# This build is configured to catch issues on PRs that would only be detected as
+# part of Heroku's build step when the code has already been merged to master.
+# The step ./bin/post_compile requires the output of `yarn build`, thus, we need
+# to build both the JS and Python builds
+[testenv:heroku]
+whitelist_externals =
+    yarn
+    post_compile
+commands_pre =
+    pip install -r requirements.txt
+    yarn install
+commands =
+    yarn heroku-postbuild
+    ./manage.py collectstatic --noinput
+    # This generates the revision and does the Brotly/Gzip compression
+    {toxinidir}/./bin/post_compile
+commands_post =
+
 [testenv:docker]
 whitelist_externals=
     docker-compose
-commands_pre = 
+commands_pre =
     docker-compose build
 commands =
     docker-compose run backend bash -c "pytest --cov --cov-report=xml tests/ --runslow --ignore=tests/selenium"