Bug 1477808 Move from TravisCI to CircleCI 2.0

This commit is contained in:
Jeff Klukas 2018-08-16 16:21:55 -04:00
Родитель c9bc94dcb2
Коммит 4d6280c04f
14 изменённых файлов: 231 добавлений и 119 удалений

133
.circleci/config.yml Normal file
Просмотреть файл

@ -0,0 +1,133 @@
####################
# CircleCI configuration reference:
# https://circleci.com/docs/2.0/configuration-reference
####################
# CircleCI built-in environment variables:
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
####################
####################
# Templates: see "anchors" in https://learnxinyminutes.com/docs/yaml/
####################
# Note: At time of writing, tests run in ~2 minutes and caching dependencies
# between runs didn't have any appreciable benefit, so we don't use
# save_cache or restore_cache steps. If build times increase substantially,
# consider using Pipenv and the cache configuration discussed in
# https://circleci.com/docs/2.0/caching/#pip-python
test_settings: &test_settings
steps:
- checkout
- run:
name: Install system packages
command: |
# python-snappy compression relies on C bindings to libsnappy-dev.
# pyspark needs to run Java, so we install openjdk.
apt-get update
apt-get install -y libsnappy-dev openjdk-8-jre-headless
- run:
name: Run tests
command: |
pip install tox
tox -e $CIRCLE_JOB
- run:
name: Submit code coverage data
command: |
# Skip this step if running via the CircleCI local CLI.
[ -z "$CIRCLE_BUILD_NUM" ] && exit 0
# Activate the virtualenv so that codecov can find the 'coverage' executable.
source .tox/$CIRCLE_JOB/bin/activate
# Upload to codecov and flag (-F) with the name of the job.
bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB
####################
# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
####################
version: 2
jobs:
py27:
<<: *test_settings
docker:
- image: python:2.7
py35:
<<: *test_settings
docker:
- image: python:3.5
py36:
<<: *test_settings
docker:
- image: python:3.6
# We don't actually add this job to the workflow yet due to issues with
# pyspark <= 2.3.1 running on Python 3.7; see:
# https://issues.apache.org/jira/browse/SPARK-24739
py37:
<<: *test_settings
docker:
- image: python:3.7
lint:
docker:
- image: python:3.6
steps:
- checkout
- run:
name: Run tests
command: |
pip install flake8
flake8 moztelemetry/ tests/
# Only runs when a tag starting with 'v' is place on the repository;
# see the workflows section below for trigger logic.
deploy:
docker:
- image: python:3.6
steps:
- checkout
- run:
name: Install deployment tools
command: |
pip install --upgrade setuptools wheel twine
- run:
name: Create the distribution files
command: |
python setup.py sdist bdist_wheel
- run:
name: Upload to PyPI
command: |
# Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at:
# https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars
# For more on twine, see:
# https://twine.readthedocs.io/en/latest/
twine upload dist/*
####################
# Workflows: see https://circleci.com/docs/2.0/workflows/
####################
workflows:
version: 2
build:
jobs:
- py27
- py35
- py36
- lint
tagged-deploy:
jobs:
- deploy:
filters:
branches:
# Ignore all branches; this workflow should only run for tags.
ignore: /.*/
tags:
only: /^v.*/

5
.dockerignore Normal file
Просмотреть файл

@ -0,0 +1,5 @@
# Files/directories matching the patterns here will not be included
# in the Docker context when you run `docker build`.
# The tox caches can be large; we ignore them to speed up builds.
.*tox/

4
.gitignore поставляемый
Просмотреть файл

@ -15,3 +15,7 @@
python_moztelemetry.egg-info/
docs/_build/
coverage.xml
build/
dist/
.coverage*
.*tox/

Просмотреть файл

@ -1,27 +0,0 @@
env:
- PYTHON_VERSION=2.7
- PYTHON_VERSION=3.5
- PYTHON_VERSION=3.6
sudo: required
language: python
services:
- docker
before_install:
- env | grep TRAVIS > .travis-env
- docker build -t moztelemetry_docker:python-$PYTHON_VERSION --build-arg PYTHON_VERSION=$PYTHON_VERSION .
script:
# mount coverage.xml as a file into the container to retrieve coverage report
- touch coverage.xml
- docker run --env-file .travis-env -v $PWD/coverage.xml:/python_moztelemetry/coverage.xml moztelemetry_docker:python-$PYTHON_VERSION ./runtests.sh -v --timeout=120 tests
after_success:
- bash <(curl -s https://codecov.io/bash) -F python${PYTHON_VERSION//.}
deploy:
provider: pypi
user: fbertsch
distributions: sdist bdist_wheel
password:
secure: "XbQWOjscb2l3c3PuTXsOKTh0dTHStNgBPt8hUR5vilpJbjnPJunUN8xt9JsHnJ/BSj5buEvk5/sVG+2RWTzavEluSGgmVTXvsEd3OtD8dhQ326NoH846H1gMWEPn0g+1jTDtZNuWfoWy14/RcOEqZRJ93aLos+0OzyWh5tB1euEVPVvpkhrAchIWNpT9UcWM99YjOfuAhtdTc2KJs5V0G0uUiQ5umUKOOwxbbu3FZOm/tLHLW4yrCE3tJ3KgEO9naY35jBrQ3fLlSj9JXJ5XA2zIleG5kle2j1Xw95RiTP+B/h5MYZv0XRDVdpWXCWJJ7ZgUogsPFXv70ekpUhvsens2gtccN6bGsLaRA+V02wMeObj8nNPTyicxwYVnjkgDNvuMuIWbz8wgD9qamIXWeRT7herMJZWYL2Jj+UVsPb2qzcXlcxfO1K3UawWx3BcDTDfd5uaO/k8KqtvERlcLRka7Yn2DBibndKxA38zWR6Bdv4ixEfQaRsborFMU2dUAQEKZvUzuUo3WMEi0r66BlzCKqOaroSP6JkkSn4YcLMxT0dWNGgktJRd0lJCCMae1IlKcRoxVqCLdukqKS9zNAi2/R3WdZ5np2eVa7BiPD7097LgnzVU0gC1Tl5NmtgordhAhoTZhGoe0/LSUl28EDX+z3EOajQuNseuMxKre5eM="
on:
tags: true
repo: mozilla/python_moztelemetry
condition: $PYTHON_VERSION = 3.6

Просмотреть файл

@ -1,41 +1,9 @@
FROM openjdk:8
# This Dockerfile supports the bin/test script for local testing.
# Make sure any changes here stay in sync with .circleci/config.yml
# so local testing and CI are comparable.
# install gcc
RUN apt-get update --fix-missing && \
apt-get install -y g++
ARG PYTHON_VERSION=3.6
FROM python:$PYTHON_VERSION
# setup conda environment
# temporary workaround, pin miniconda version until it's fixed.
RUN echo "export PATH=/miniconda/bin:${PATH}" > /etc/profile.d/conda.sh && \
wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.21-Linux-x86_64.sh -O miniconda.sh && \
/bin/bash miniconda.sh -b -p /miniconda && \
rm miniconda.sh
ENV PATH /miniconda/bin:${PATH}
RUN hash -r && \
conda config --set always_yes yes --set changeps1 no && \
# TODO: uncomment \
# RUN conda update -q conda && \
conda info -a # Useful for debugging any issues with conda
# build + activate conda environment
COPY ./environment.yml /python_moztelemetry/
ARG PYTHON_VERSION=2.7
RUN echo "- python=$PYTHON_VERSION" >> /python_moztelemetry/environment.yml && \
conda env create -f /python_moztelemetry/environment.yml
# this is roughly equivalent to activating the conda environment
ENV PATH="/miniconda/envs/test-environment/bin:${PATH}"
WORKDIR /python_moztelemetry
# we need to explicitly install pytest and dependencies so spark
# can pick them up
RUN pip install 'pytest>=3'
# This will invalidate the cache if something changes in python_moztelemetry.
COPY . /python_moztelemetry
# install moztelemetry specific deps into conda env
RUN pip install .[test]
RUN apt-get update && apt-get install -y libsnappy-dev openjdk-8-jre-headless
RUN pip install tox

Просмотреть файл

@ -1,5 +1,5 @@
# python_moztelemetry [![Build Status](https://travis-ci.org/mozilla/python_moztelemetry.svg?branch=master)](https://travis-ci.org/mozilla/python_moztelemetry) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)
# python_moztelemetry [![CircleCI Build Status](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master.svg?style=svg)](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)
Spark bindings for Mozilla Telemetry
@ -24,18 +24,25 @@ Note that this file was formerly called histogram_tools.py and was renamed in Bu
## Testing locally
To test/debug this package locally, the recommended procedure is to build a
docker image with the appropriate dependencies, then execute the unit
tests inside it:
To test/debug this package locally, you can run exactly the job that
CircleCI runs for continuous integration by
[installing the CircleCI local CLI](https://circleci.com/docs/2.0/local-cli/#installing-the-circleci-local-cli-on-macos-and-linux-distros)
and invoking:
```bash
docker build -t moztelemetry_docker .
./runtests.sh # will run tests inside docker container
circleci build --job py36
```
You can also run a subset of the tests by passing arguments to `runtests.sh`:
See [.circleci/config.yml] for the other configured job names
(for running tests on different python versions).
The above process takes a few minutes to run every time, so there
is also a `bin/test` script that builds a docker image and
python environment (both of which are cached locally) and allows
you to run a subset of tests. Here are some sample invocations:
```bash
./runtests.sh -ktest_unpack # runs only tests with key "test_unpack"
./runtests.sh tests/heka # runs only tests in tests/heka
./bin/test tests/ -k test_unpack # runs only tests with key "test_unpack"
./bin/test tests/heka/ # runs only tests in tests/heka
PYTHON_VERSION=2.7 ./bin/test # specify a python version
```

28
bin/test Executable file
Просмотреть файл

@ -0,0 +1,28 @@
#!/bin/bash
# Abort immediately on any failure.
set -e
# Move to the root of the git repository.
cd $(git rev-parse --show-toplevel)
# Set default variables if not already present in the environment.
: ${PYTHON_VERSION:=3.6}
: ${WORKDIR:=.dockertox} # Prevents the circleci cli from picking up the env
: ${TOXENV:="py${PYTHON_VERSION/./}"}
# Set some derived variables.
TOXENV="py${PYTHON_VERSION/./}"
# Clean all python cache files, since they may confuse the docker environment.
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
# If nothing has changed, this build step will hit cache and run quickly.
docker build -t moztelemetry:"$TOXENV" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
# We mount the current directory to the container and run tox inside it.
docker run -it --rm \
--volume "$(pwd)":/python_moztelemetry \
--workdir /python_moztelemetry \
moztelemetry:"$TOXENV" \
tox --workdir $WORKDIR -e "$TOXENV" -- $@

Просмотреть файл

@ -1,8 +0,0 @@
name: test-environment
dependencies:
- pandas
- pyspark
- python-snappy
- snappy
- scipy
- gcc

Просмотреть файл

@ -1,3 +0,0 @@
[pytest]
norecursedirs = .git .*
addopts = -rsxX --showlocals --tb=native --flake8 moztelemetry

Просмотреть файл

@ -1,5 +1,3 @@
conda:
file: environment.yml
python:
version: 2.7
setup_py_install: true

Просмотреть файл

@ -1,26 +0,0 @@
#!/bin/bash
# abort immediately on any failure
set -e
PYTHON_VERSION="${PYTHON_VERSION:-2.7}"
REBUILD_DOCKER="${REBUILD_DOCKER:-false}"
# if we are not inside the docker container, run this command *inside* the
# docker container
if [ ! -f /.dockerenv ]; then
if $REBUILD_DOCKER || ! docker images moztelemetry_docker:python-"$PYTHON_VERSION" | grep -q moztelemetry_docker; then
docker build -t moztelemetry_docker:python-"$PYTHON_VERSION" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
fi
docker run -t -i -v "$PWD":/python_moztelemetry moztelemetry_docker:python-"$PYTHON_VERSION" ./runtests.sh "$@"
exit $?
fi
# Run tests
if [ $# -gt 0 ]; then
ARGS="$@"
coverage run --source=moztelemetry setup.py test --addopts "${ARGS}"
else
coverage run --source=moztelemetry setup.py test
fi
coverage xml

Просмотреть файл

@ -4,8 +4,10 @@ test=pytest
[flake8]
max-line-length=100
exclude=
.svn,CVS,.bzr,.hg,.git,__pycache__,.*tox,.eggs,*.egg,
moztelemetry/parse_histograms.py,
moztelemetry/heka/message_pb2.py
moztelemetry/heka/message_pb2.py,
.*tox
# * E226: allow whitespace around arithmetic operators
# * E221,E241,E251: ignore spaces around keyword arguments and dict entries,
@ -14,7 +16,8 @@ exclude=
ignore=E221,E226,E241,E251,E501
[coverage:run]
omit=
source = moztelemetry/
omit =
moztelemetry/shared_telemetry_utils.py,
moztelemetry/parse_histograms.py,
moztelemetry/heka/message_pb2.py

Просмотреть файл

@ -6,6 +6,9 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup, find_packages
tests_require = ['mock', 'pytest-timeout', 'moto', 'responses',
'scipy', 'pyspark', 'pytest', 'pytest-cov']
setup(
name='python_moztelemetry',
use_scm_version=True,
@ -21,8 +24,7 @@ setup(
'PyYAML', 'python-snappy'],
setup_requires=['pytest-runner', 'setuptools_scm'],
extras_require={
'test': ['mock', 'pytest-timeout', 'moto<=1.1.22', 'responses',
'scipy', 'pytest-flake8', 'pytest', 'pytest-cov'],
'testing': tests_require,
},
tests_require=['python_moztelemetry[test]'],
tests_require=tests_require,
)

28
tox.ini Normal file
Просмотреть файл

@ -0,0 +1,28 @@
# Tox (https://tox.readthedocs.io/) is a tool for maintaining
# multiple virtualenv environments for different python versions.
# This file is referenced when tox is invoked in bin/test or .circleci/config.yml
[tox]
envlist = py36 # CircleCI jobs override the envlist; see .circleci/config.yml
[pytest]
addopts =
-rsxX
--verbose
--showlocals
--tb=native
--timeout=120
--capture=no
--cov
[testenv]
skipsdist = True
usedevelop = True
extras = testing
commands = pytest {posargs:tests/}
[testenv:lint]
deps =
flake8
commands =
flake8 moztelemetry/ tests/