Bug 1477808 Move from TravisCI to CircleCI 2.0

2018-08-16 16:21:55 -04:00 · 2018-08-16 16:21:55 -04:00 · 4d6280c04f
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -0,0 +1,133 @@
+####################
+# CircleCI configuration reference:
+#   https://circleci.com/docs/2.0/configuration-reference
+####################
+# CircleCI built-in environment variables:
+#   https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+####################
+
+
+####################
+# Templates: see "anchors" in https://learnxinyminutes.com/docs/yaml/
+####################
+
+# Note: At time of writing, tests run in ~2 minutes and caching dependencies
+# between runs didn't have any appreciable benefit, so we don't use
+# save_cache or restore_cache steps. If build times increase substantially,
+# consider using Pipenv and the cache configuration discussed in
+# https://circleci.com/docs/2.0/caching/#pip-python
+
+test_settings: &test_settings
+  steps:
+    - checkout
+    - run:
+        name: Install system packages
+        command: |
+          # python-snappy compression relies on C bindings to libsnappy-dev.
+          # pyspark needs to run Java, so we install openjdk.
+          apt-get update
+          apt-get install -y libsnappy-dev openjdk-8-jre-headless
+    - run:
+        name: Run tests
+        command: |
+          pip install tox
+          tox -e $CIRCLE_JOB
+    - run:
+        name: Submit code coverage data
+        command: |
+          # Skip this step if running via the CircleCI local CLI.
+          [ -z "$CIRCLE_BUILD_NUM" ] && exit 0
+          # Activate the virtualenv so that codecov can find the 'coverage' executable.
+          source .tox/$CIRCLE_JOB/bin/activate
+          # Upload to codecov and flag (-F) with the name of the job.
+          bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB
+
+
+
+####################
+# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
+####################
+
+version: 2
+jobs:
+
+  py27:
+    <<: *test_settings
+    docker:
+      - image: python:2.7
+
+  py35:
+    <<: *test_settings
+    docker:
+      - image: python:3.5
+
+  py36:
+    <<: *test_settings
+    docker:
+      - image: python:3.6
+
+  # We don't actually add this job to the workflow yet due to issues with
+  # pyspark <= 2.3.1 running on Python 3.7; see:
+  # https://issues.apache.org/jira/browse/SPARK-24739
+  py37:
+    <<: *test_settings
+    docker:
+      - image: python:3.7
+
+  lint:
+    docker:
+      - image: python:3.6
+    steps:
+    - checkout
+    - run:
+        name: Run tests
+        command: |
+          pip install flake8
+          flake8 moztelemetry/ tests/
+
+  # Only runs when a tag starting with 'v' is place on the repository;
+  # see the workflows section below for trigger logic.
+  deploy:
+    docker:
+      - image: python:3.6
+    steps:
+      - checkout
+      - run:
+          name: Install deployment tools
+          command: |
+            pip install --upgrade setuptools wheel twine
+      - run:
+          name: Create the distribution files
+          command: |
+            python setup.py sdist bdist_wheel
+      - run:
+          name: Upload to PyPI
+          command: |
+            # Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at:
+            #   https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars
+            # For more on twine, see:
+            #   https://twine.readthedocs.io/en/latest/
+            twine upload dist/*
+
+
+####################
+# Workflows: see https://circleci.com/docs/2.0/workflows/
+####################
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - py27
+      - py35
+      - py36
+      - lint
+  tagged-deploy:
+    jobs:
+      - deploy:
+          filters:
+            branches:
+              # Ignore all branches; this workflow should only run for tags.
+              ignore: /.*/
+            tags:
+              only: /^v.*/
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,5 @@
+# Files/directories matching the patterns here will not be included
+# in the Docker context when you run `docker build`.
+
+# The tox caches can be large; we ignore them to speed up builds.
+.*tox/
--- a/.gitignore
+++ b/.gitignore
@ -15,3 +15,7 @@
 python_moztelemetry.egg-info/
 docs/_build/
 coverage.xml
+build/
+dist/
+.coverage*
+.*tox/
--- a/.travis.yml
+++ b/.travis.yml
@ -1,27 +0,0 @@
-env:
-  - PYTHON_VERSION=2.7
-  - PYTHON_VERSION=3.5
-  - PYTHON_VERSION=3.6
-sudo: required
-language: python
-services:
-  - docker
-before_install:
-  - env | grep TRAVIS > .travis-env
-  - docker build -t moztelemetry_docker:python-$PYTHON_VERSION --build-arg PYTHON_VERSION=$PYTHON_VERSION .
-script:
-  # mount coverage.xml as a file into the container to retrieve coverage report
-  - touch coverage.xml
-  - docker run --env-file .travis-env -v $PWD/coverage.xml:/python_moztelemetry/coverage.xml moztelemetry_docker:python-$PYTHON_VERSION ./runtests.sh -v --timeout=120 tests
-after_success:
-  - bash <(curl -s https://codecov.io/bash) -F python${PYTHON_VERSION//.}
-deploy:
-  provider: pypi
-  user: fbertsch
-  distributions: sdist bdist_wheel
-  password:
-    secure: "XbQWOjscb2l3c3PuTXsOKTh0dTHStNgBPt8hUR5vilpJbjnPJunUN8xt9JsHnJ/BSj5buEvk5/sVG+2RWTzavEluSGgmVTXvsEd3OtD8dhQ326NoH846H1gMWEPn0g+1jTDtZNuWfoWy14/RcOEqZRJ93aLos+0OzyWh5tB1euEVPVvpkhrAchIWNpT9UcWM99YjOfuAhtdTc2KJs5V0G0uUiQ5umUKOOwxbbu3FZOm/tLHLW4yrCE3tJ3KgEO9naY35jBrQ3fLlSj9JXJ5XA2zIleG5kle2j1Xw95RiTP+B/h5MYZv0XRDVdpWXCWJJ7ZgUogsPFXv70ekpUhvsens2gtccN6bGsLaRA+V02wMeObj8nNPTyicxwYVnjkgDNvuMuIWbz8wgD9qamIXWeRT7herMJZWYL2Jj+UVsPb2qzcXlcxfO1K3UawWx3BcDTDfd5uaO/k8KqtvERlcLRka7Yn2DBibndKxA38zWR6Bdv4ixEfQaRsborFMU2dUAQEKZvUzuUo3WMEi0r66BlzCKqOaroSP6JkkSn4YcLMxT0dWNGgktJRd0lJCCMae1IlKcRoxVqCLdukqKS9zNAi2/R3WdZ5np2eVa7BiPD7097LgnzVU0gC1Tl5NmtgordhAhoTZhGoe0/LSUl28EDX+z3EOajQuNseuMxKre5eM="
-  on:
-    tags: true
-    repo: mozilla/python_moztelemetry
-    condition: $PYTHON_VERSION = 3.6
--- a/46
+++ b/46
@ -1,41 +1,9 @@
-FROM openjdk:8
+# This Dockerfile supports the bin/test script for local testing.
+# Make sure any changes here stay in sync with .circleci/config.yml
+# so local testing and CI are comparable.

-# install gcc
-RUN apt-get update --fix-missing && \
-    apt-get install -y g++
+ARG PYTHON_VERSION=3.6
+FROM python:$PYTHON_VERSION

-# setup conda environment
-# temporary workaround, pin miniconda version until it's fixed.
-RUN echo "export PATH=/miniconda/bin:${PATH}" > /etc/profile.d/conda.sh && \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.21-Linux-x86_64.sh -O miniconda.sh && \
-    /bin/bash miniconda.sh -b -p /miniconda && \
-    rm miniconda.sh
-
-ENV PATH /miniconda/bin:${PATH}
-
-RUN hash -r && \
-    conda config --set always_yes yes --set changeps1 no && \
-    # TODO: uncomment \
-    # RUN conda update -q conda && \
-    conda info -a # Useful for debugging any issues with conda
-
-# build + activate conda environment
-COPY ./environment.yml /python_moztelemetry/
-ARG PYTHON_VERSION=2.7
-RUN echo "- python=$PYTHON_VERSION" >> /python_moztelemetry/environment.yml && \
-    conda env create -f /python_moztelemetry/environment.yml
-
-# this is roughly equivalent to activating the conda environment
-ENV PATH="/miniconda/envs/test-environment/bin:${PATH}"
-
-WORKDIR /python_moztelemetry
-
-# we need to explicitly install pytest and dependencies so spark
-# can pick them up
-RUN pip install 'pytest>=3'
-
-# This will invalidate the cache if something changes in python_moztelemetry.
-COPY . /python_moztelemetry
-
-# install moztelemetry specific deps into conda env
-RUN pip install .[test]
+RUN apt-get update && apt-get install -y libsnappy-dev openjdk-8-jre-headless
+RUN pip install tox
--- a/README.md
+++ b/README.md
@ -1,5 +1,5 @@

-# python_moztelemetry [![Build Status](https://travis-ci.org/mozilla/python_moztelemetry.svg?branch=master)](https://travis-ci.org/mozilla/python_moztelemetry) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)
+# python_moztelemetry [![CircleCI Build Status](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master.svg?style=svg)](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)

 Spark bindings for Mozilla Telemetry

@ -24,18 +24,25 @@ Note that this file was formerly called histogram_tools.py and was renamed in Bu

 ## Testing locally

-To test/debug this package locally, the recommended procedure is to build a
-docker image with the appropriate dependencies, then execute the unit
-tests inside it:
+To test/debug this package locally, you can run exactly the job that
+CircleCI runs for continuous integration by
+[installing the CircleCI local CLI](https://circleci.com/docs/2.0/local-cli/#installing-the-circleci-local-cli-on-macos-and-linux-distros)
+and invoking:

 ```bash
-docker build -t moztelemetry_docker .
-./runtests.sh # will run tests inside docker container
+circleci build --job py36
 ```

-You can also run a subset of the tests by passing arguments to `runtests.sh`:
+See [.circleci/config.yml] for the other configured job names
+(for running tests on different python versions).
+
+The above process takes a few minutes to run every time, so there
+is also a `bin/test` script that builds a docker image and
+python environment (both of which are cached locally) and allows
+you to run a subset of tests. Here are some sample invocations:

 ```bash
-./runtests.sh -ktest_unpack # runs only tests with key "test_unpack"
-./runtests.sh tests/heka # runs only tests in tests/heka
+./bin/test tests/ -k test_unpack  # runs only tests with key "test_unpack"
+./bin/test tests/heka/            # runs only tests in tests/heka
+PYTHON_VERSION=2.7 ./bin/test     # specify a python version
 ```
--- a/bin/test
+++ b/bin/test
@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Abort immediately on any failure.
+set -e
+
+# Move to the root of the git repository.
+cd $(git rev-parse --show-toplevel)
+
+# Set default variables if not already present in the environment.
+: ${PYTHON_VERSION:=3.6}
+: ${WORKDIR:=.dockertox}  # Prevents the circleci cli from picking up the env
+: ${TOXENV:="py${PYTHON_VERSION/./}"}
+
+# Set some derived variables.
+TOXENV="py${PYTHON_VERSION/./}"
+
+# Clean all python cache files, since they may confuse the docker environment.
+find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
+
+# If nothing has changed, this build step will hit cache and run quickly.
+docker build -t moztelemetry:"$TOXENV" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
+
+# We mount the current directory to the container and run tox inside it.
+docker run -it --rm \
+       --volume "$(pwd)":/python_moztelemetry \
+       --workdir /python_moztelemetry \
+       moztelemetry:"$TOXENV" \
+       tox --workdir $WORKDIR -e "$TOXENV" -- $@
--- a/environment.yml
+++ b/environment.yml
@ -1,8 +0,0 @@
-name: test-environment
-dependencies:
- pandas
- pyspark
- python-snappy
- snappy
- scipy
- gcc
--- a/pytest.ini
+++ b/pytest.ini
@ -1,3 +0,0 @@
-[pytest]
-norecursedirs = .git .*
-addopts = -rsxX --showlocals --tb=native --flake8 moztelemetry
--- a/readthedocs.yml
+++ b/readthedocs.yml
@ -1,5 +1,3 @@
-conda:
-    file: environment.yml
 python:
    version: 2.7
    setup_py_install: true
--- a/runtests.sh
+++ b/runtests.sh
@ -1,26 +0,0 @@
-#!/bin/bash
-
-# abort immediately on any failure
-set -e
-
-PYTHON_VERSION="${PYTHON_VERSION:-2.7}"
-REBUILD_DOCKER="${REBUILD_DOCKER:-false}"
-
-# if we are not inside the docker container, run this command *inside* the
-# docker container
-if [ ! -f /.dockerenv ]; then
-    if $REBUILD_DOCKER || ! docker images moztelemetry_docker:python-"$PYTHON_VERSION" | grep -q moztelemetry_docker; then
-        docker build -t moztelemetry_docker:python-"$PYTHON_VERSION" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
-    fi
-    docker run -t -i -v "$PWD":/python_moztelemetry moztelemetry_docker:python-"$PYTHON_VERSION" ./runtests.sh "$@"
-    exit $?
-fi
-
-# Run tests
-if [ $# -gt 0 ]; then
-    ARGS="$@"
-    coverage run --source=moztelemetry setup.py test --addopts "${ARGS}"
-else
-    coverage run --source=moztelemetry setup.py test
-fi
-coverage xml
--- a/setup.cfg
+++ b/setup.cfg
@ -4,8 +4,10 @@ test=pytest
 [flake8]
 max-line-length=100
 exclude=
+    .svn,CVS,.bzr,.hg,.git,__pycache__,.*tox,.eggs,*.egg,
    moztelemetry/parse_histograms.py,
-    moztelemetry/heka/message_pb2.py
+    moztelemetry/heka/message_pb2.py,
+    .*tox

 # * E226: allow whitespace around arithmetic operators
 # * E221,E241,E251: ignore spaces around keyword arguments and dict entries,
@ -14,6 +16,7 @@ exclude=
 ignore=E221,E226,E241,E251,E501

 [coverage:run]
+source = moztelemetry/
 omit =
    moztelemetry/shared_telemetry_utils.py,
    moztelemetry/parse_histograms.py,
--- a/setup.py
+++ b/setup.py
@ -6,6 +6,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from setuptools import setup, find_packages

+tests_require = ['mock', 'pytest-timeout', 'moto', 'responses',
+                 'scipy', 'pyspark', 'pytest', 'pytest-cov']
+
 setup(
    name='python_moztelemetry',
    use_scm_version=True,
@ -21,8 +24,7 @@ setup(
                      'PyYAML', 'python-snappy'],
    setup_requires=['pytest-runner', 'setuptools_scm'],
    extras_require={
-        'test': ['mock', 'pytest-timeout', 'moto<=1.1.22', 'responses',
-                 'scipy', 'pytest-flake8', 'pytest', 'pytest-cov'],
+        'testing': tests_require,
    },
-    tests_require=['python_moztelemetry[test]'],
+    tests_require=tests_require,
 )
--- a/tox.ini
+++ b/tox.ini
@ -0,0 +1,28 @@
+# Tox (https://tox.readthedocs.io/) is a tool for maintaining
+# multiple virtualenv environments for different python versions.
+# This file is referenced when tox is invoked in bin/test or .circleci/config.yml
+
+[tox]
+envlist = py36  # CircleCI jobs override the envlist; see .circleci/config.yml
+
+[pytest]
+addopts =
+    -rsxX
+    --verbose
+    --showlocals
+    --tb=native
+    --timeout=120
+    --capture=no
+    --cov
+
+[testenv]
+skipsdist = True
+usedevelop = True
+extras = testing
+commands = pytest {posargs:tests/}
+
+[testenv:lint]
+deps =
+    flake8
+commands =
+    flake8 moztelemetry/ tests/