Bug 1477808 Move from TravisCI to CircleCI 2.0
This commit is contained in:
Родитель
c9bc94dcb2
Коммит
4d6280c04f
|
@ -0,0 +1,133 @@
|
|||
####################
|
||||
# CircleCI configuration reference:
|
||||
# https://circleci.com/docs/2.0/configuration-reference
|
||||
####################
|
||||
# CircleCI built-in environment variables:
|
||||
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
|
||||
####################
|
||||
|
||||
|
||||
####################
|
||||
# Templates: see "anchors" in https://learnxinyminutes.com/docs/yaml/
|
||||
####################
|
||||
|
||||
# Note: At time of writing, tests run in ~2 minutes and caching dependencies
|
||||
# between runs didn't have any appreciable benefit, so we don't use
|
||||
# save_cache or restore_cache steps. If build times increase substantially,
|
||||
# consider using Pipenv and the cache configuration discussed in
|
||||
# https://circleci.com/docs/2.0/caching/#pip-python
|
||||
|
||||
test_settings: &test_settings
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install system packages
|
||||
command: |
|
||||
# python-snappy compression relies on C bindings to libsnappy-dev.
|
||||
# pyspark needs to run Java, so we install openjdk.
|
||||
apt-get update
|
||||
apt-get install -y libsnappy-dev openjdk-8-jre-headless
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
pip install tox
|
||||
tox -e $CIRCLE_JOB
|
||||
- run:
|
||||
name: Submit code coverage data
|
||||
command: |
|
||||
# Skip this step if running via the CircleCI local CLI.
|
||||
[ -z "$CIRCLE_BUILD_NUM" ] && exit 0
|
||||
# Activate the virtualenv so that codecov can find the 'coverage' executable.
|
||||
source .tox/$CIRCLE_JOB/bin/activate
|
||||
# Upload to codecov and flag (-F) with the name of the job.
|
||||
bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB
|
||||
|
||||
|
||||
|
||||
####################
|
||||
# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
|
||||
####################
|
||||
|
||||
version: 2
|
||||
jobs:
|
||||
|
||||
py27:
|
||||
<<: *test_settings
|
||||
docker:
|
||||
- image: python:2.7
|
||||
|
||||
py35:
|
||||
<<: *test_settings
|
||||
docker:
|
||||
- image: python:3.5
|
||||
|
||||
py36:
|
||||
<<: *test_settings
|
||||
docker:
|
||||
- image: python:3.6
|
||||
|
||||
# We don't actually add this job to the workflow yet due to issues with
|
||||
# pyspark <= 2.3.1 running on Python 3.7; see:
|
||||
# https://issues.apache.org/jira/browse/SPARK-24739
|
||||
py37:
|
||||
<<: *test_settings
|
||||
docker:
|
||||
- image: python:3.7
|
||||
|
||||
lint:
|
||||
docker:
|
||||
- image: python:3.6
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Run tests
|
||||
command: |
|
||||
pip install flake8
|
||||
flake8 moztelemetry/ tests/
|
||||
|
||||
# Only runs when a tag starting with 'v' is place on the repository;
|
||||
# see the workflows section below for trigger logic.
|
||||
deploy:
|
||||
docker:
|
||||
- image: python:3.6
|
||||
steps:
|
||||
- checkout
|
||||
- run:
|
||||
name: Install deployment tools
|
||||
command: |
|
||||
pip install --upgrade setuptools wheel twine
|
||||
- run:
|
||||
name: Create the distribution files
|
||||
command: |
|
||||
python setup.py sdist bdist_wheel
|
||||
- run:
|
||||
name: Upload to PyPI
|
||||
command: |
|
||||
# Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at:
|
||||
# https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars
|
||||
# For more on twine, see:
|
||||
# https://twine.readthedocs.io/en/latest/
|
||||
twine upload dist/*
|
||||
|
||||
|
||||
####################
|
||||
# Workflows: see https://circleci.com/docs/2.0/workflows/
|
||||
####################
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- py27
|
||||
- py35
|
||||
- py36
|
||||
- lint
|
||||
tagged-deploy:
|
||||
jobs:
|
||||
- deploy:
|
||||
filters:
|
||||
branches:
|
||||
# Ignore all branches; this workflow should only run for tags.
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: /^v.*/
|
|
@ -0,0 +1,5 @@
|
|||
# Files/directories matching the patterns here will not be included
|
||||
# in the Docker context when you run `docker build`.
|
||||
|
||||
# The tox caches can be large; we ignore them to speed up builds.
|
||||
.*tox/
|
|
@ -15,3 +15,7 @@
|
|||
python_moztelemetry.egg-info/
|
||||
docs/_build/
|
||||
coverage.xml
|
||||
build/
|
||||
dist/
|
||||
.coverage*
|
||||
.*tox/
|
||||
|
|
27
.travis.yml
27
.travis.yml
|
@ -1,27 +0,0 @@
|
|||
env:
|
||||
- PYTHON_VERSION=2.7
|
||||
- PYTHON_VERSION=3.5
|
||||
- PYTHON_VERSION=3.6
|
||||
sudo: required
|
||||
language: python
|
||||
services:
|
||||
- docker
|
||||
before_install:
|
||||
- env | grep TRAVIS > .travis-env
|
||||
- docker build -t moztelemetry_docker:python-$PYTHON_VERSION --build-arg PYTHON_VERSION=$PYTHON_VERSION .
|
||||
script:
|
||||
# mount coverage.xml as a file into the container to retrieve coverage report
|
||||
- touch coverage.xml
|
||||
- docker run --env-file .travis-env -v $PWD/coverage.xml:/python_moztelemetry/coverage.xml moztelemetry_docker:python-$PYTHON_VERSION ./runtests.sh -v --timeout=120 tests
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash) -F python${PYTHON_VERSION//.}
|
||||
deploy:
|
||||
provider: pypi
|
||||
user: fbertsch
|
||||
distributions: sdist bdist_wheel
|
||||
password:
|
||||
secure: "XbQWOjscb2l3c3PuTXsOKTh0dTHStNgBPt8hUR5vilpJbjnPJunUN8xt9JsHnJ/BSj5buEvk5/sVG+2RWTzavEluSGgmVTXvsEd3OtD8dhQ326NoH846H1gMWEPn0g+1jTDtZNuWfoWy14/RcOEqZRJ93aLos+0OzyWh5tB1euEVPVvpkhrAchIWNpT9UcWM99YjOfuAhtdTc2KJs5V0G0uUiQ5umUKOOwxbbu3FZOm/tLHLW4yrCE3tJ3KgEO9naY35jBrQ3fLlSj9JXJ5XA2zIleG5kle2j1Xw95RiTP+B/h5MYZv0XRDVdpWXCWJJ7ZgUogsPFXv70ekpUhvsens2gtccN6bGsLaRA+V02wMeObj8nNPTyicxwYVnjkgDNvuMuIWbz8wgD9qamIXWeRT7herMJZWYL2Jj+UVsPb2qzcXlcxfO1K3UawWx3BcDTDfd5uaO/k8KqtvERlcLRka7Yn2DBibndKxA38zWR6Bdv4ixEfQaRsborFMU2dUAQEKZvUzuUo3WMEi0r66BlzCKqOaroSP6JkkSn4YcLMxT0dWNGgktJRd0lJCCMae1IlKcRoxVqCLdukqKS9zNAi2/R3WdZ5np2eVa7BiPD7097LgnzVU0gC1Tl5NmtgordhAhoTZhGoe0/LSUl28EDX+z3EOajQuNseuMxKre5eM="
|
||||
on:
|
||||
tags: true
|
||||
repo: mozilla/python_moztelemetry
|
||||
condition: $PYTHON_VERSION = 3.6
|
46
Dockerfile
46
Dockerfile
|
@ -1,41 +1,9 @@
|
|||
FROM openjdk:8
|
||||
# This Dockerfile supports the bin/test script for local testing.
|
||||
# Make sure any changes here stay in sync with .circleci/config.yml
|
||||
# so local testing and CI are comparable.
|
||||
|
||||
# install gcc
|
||||
RUN apt-get update --fix-missing && \
|
||||
apt-get install -y g++
|
||||
ARG PYTHON_VERSION=3.6
|
||||
FROM python:$PYTHON_VERSION
|
||||
|
||||
# setup conda environment
|
||||
# temporary workaround, pin miniconda version until it's fixed.
|
||||
RUN echo "export PATH=/miniconda/bin:${PATH}" > /etc/profile.d/conda.sh && \
|
||||
wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.21-Linux-x86_64.sh -O miniconda.sh && \
|
||||
/bin/bash miniconda.sh -b -p /miniconda && \
|
||||
rm miniconda.sh
|
||||
|
||||
ENV PATH /miniconda/bin:${PATH}
|
||||
|
||||
RUN hash -r && \
|
||||
conda config --set always_yes yes --set changeps1 no && \
|
||||
# TODO: uncomment \
|
||||
# RUN conda update -q conda && \
|
||||
conda info -a # Useful for debugging any issues with conda
|
||||
|
||||
# build + activate conda environment
|
||||
COPY ./environment.yml /python_moztelemetry/
|
||||
ARG PYTHON_VERSION=2.7
|
||||
RUN echo "- python=$PYTHON_VERSION" >> /python_moztelemetry/environment.yml && \
|
||||
conda env create -f /python_moztelemetry/environment.yml
|
||||
|
||||
# this is roughly equivalent to activating the conda environment
|
||||
ENV PATH="/miniconda/envs/test-environment/bin:${PATH}"
|
||||
|
||||
WORKDIR /python_moztelemetry
|
||||
|
||||
# we need to explicitly install pytest and dependencies so spark
|
||||
# can pick them up
|
||||
RUN pip install 'pytest>=3'
|
||||
|
||||
# This will invalidate the cache if something changes in python_moztelemetry.
|
||||
COPY . /python_moztelemetry
|
||||
|
||||
# install moztelemetry specific deps into conda env
|
||||
RUN pip install .[test]
|
||||
RUN apt-get update && apt-get install -y libsnappy-dev openjdk-8-jre-headless
|
||||
RUN pip install tox
|
||||
|
|
25
README.md
25
README.md
|
@ -1,5 +1,5 @@
|
|||
|
||||
# python_moztelemetry [![Build Status](https://travis-ci.org/mozilla/python_moztelemetry.svg?branch=master)](https://travis-ci.org/mozilla/python_moztelemetry) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)
|
||||
# python_moztelemetry [![CircleCI Build Status](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master.svg?style=svg)](https://circleci.com/gh/mozilla/python_moztelemetry/tree/master) [![Documentation Status](http://readthedocs.org/projects/python_moztelemetry/badge/?version=latest)](https://python_moztelemetry.readthedocs.io/?badge=latest) [![Updates](https://pyup.io/repos/github/mozilla/python_moztelemetry/shield.svg)](https://pyup.io/repos/github/mozilla/python_moztelemetry/) [![codecov.io](https://codecov.io/github/mozilla/python_moztelemetry/coverage.svg?branch=master)](https://codecov.io/github/mozilla/python_moztelemetry?branch=master)
|
||||
|
||||
Spark bindings for Mozilla Telemetry
|
||||
|
||||
|
@ -24,18 +24,25 @@ Note that this file was formerly called histogram_tools.py and was renamed in Bu
|
|||
|
||||
## Testing locally
|
||||
|
||||
To test/debug this package locally, the recommended procedure is to build a
|
||||
docker image with the appropriate dependencies, then execute the unit
|
||||
tests inside it:
|
||||
To test/debug this package locally, you can run exactly the job that
|
||||
CircleCI runs for continuous integration by
|
||||
[installing the CircleCI local CLI](https://circleci.com/docs/2.0/local-cli/#installing-the-circleci-local-cli-on-macos-and-linux-distros)
|
||||
and invoking:
|
||||
|
||||
```bash
|
||||
docker build -t moztelemetry_docker .
|
||||
./runtests.sh # will run tests inside docker container
|
||||
circleci build --job py36
|
||||
```
|
||||
|
||||
You can also run a subset of the tests by passing arguments to `runtests.sh`:
|
||||
See [.circleci/config.yml] for the other configured job names
|
||||
(for running tests on different python versions).
|
||||
|
||||
The above process takes a few minutes to run every time, so there
|
||||
is also a `bin/test` script that builds a docker image and
|
||||
python environment (both of which are cached locally) and allows
|
||||
you to run a subset of tests. Here are some sample invocations:
|
||||
|
||||
```bash
|
||||
./runtests.sh -ktest_unpack # runs only tests with key "test_unpack"
|
||||
./runtests.sh tests/heka # runs only tests in tests/heka
|
||||
./bin/test tests/ -k test_unpack # runs only tests with key "test_unpack"
|
||||
./bin/test tests/heka/ # runs only tests in tests/heka
|
||||
PYTHON_VERSION=2.7 ./bin/test # specify a python version
|
||||
```
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Abort immediately on any failure.
|
||||
set -e
|
||||
|
||||
# Move to the root of the git repository.
|
||||
cd $(git rev-parse --show-toplevel)
|
||||
|
||||
# Set default variables if not already present in the environment.
|
||||
: ${PYTHON_VERSION:=3.6}
|
||||
: ${WORKDIR:=.dockertox} # Prevents the circleci cli from picking up the env
|
||||
: ${TOXENV:="py${PYTHON_VERSION/./}"}
|
||||
|
||||
# Set some derived variables.
|
||||
TOXENV="py${PYTHON_VERSION/./}"
|
||||
|
||||
# Clean all python cache files, since they may confuse the docker environment.
|
||||
find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf
|
||||
|
||||
# If nothing has changed, this build step will hit cache and run quickly.
|
||||
docker build -t moztelemetry:"$TOXENV" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
|
||||
|
||||
# We mount the current directory to the container and run tox inside it.
|
||||
docker run -it --rm \
|
||||
--volume "$(pwd)":/python_moztelemetry \
|
||||
--workdir /python_moztelemetry \
|
||||
moztelemetry:"$TOXENV" \
|
||||
tox --workdir $WORKDIR -e "$TOXENV" -- $@
|
|
@ -1,8 +0,0 @@
|
|||
name: test-environment
|
||||
dependencies:
|
||||
- pandas
|
||||
- pyspark
|
||||
- python-snappy
|
||||
- snappy
|
||||
- scipy
|
||||
- gcc
|
|
@ -1,3 +0,0 @@
|
|||
[pytest]
|
||||
norecursedirs = .git .*
|
||||
addopts = -rsxX --showlocals --tb=native --flake8 moztelemetry
|
|
@ -1,5 +1,3 @@
|
|||
conda:
|
||||
file: environment.yml
|
||||
python:
|
||||
version: 2.7
|
||||
setup_py_install: true
|
||||
|
|
26
runtests.sh
26
runtests.sh
|
@ -1,26 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# abort immediately on any failure
|
||||
set -e
|
||||
|
||||
PYTHON_VERSION="${PYTHON_VERSION:-2.7}"
|
||||
REBUILD_DOCKER="${REBUILD_DOCKER:-false}"
|
||||
|
||||
# if we are not inside the docker container, run this command *inside* the
|
||||
# docker container
|
||||
if [ ! -f /.dockerenv ]; then
|
||||
if $REBUILD_DOCKER || ! docker images moztelemetry_docker:python-"$PYTHON_VERSION" | grep -q moztelemetry_docker; then
|
||||
docker build -t moztelemetry_docker:python-"$PYTHON_VERSION" --build-arg PYTHON_VERSION="$PYTHON_VERSION" .
|
||||
fi
|
||||
docker run -t -i -v "$PWD":/python_moztelemetry moztelemetry_docker:python-"$PYTHON_VERSION" ./runtests.sh "$@"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# Run tests
|
||||
if [ $# -gt 0 ]; then
|
||||
ARGS="$@"
|
||||
coverage run --source=moztelemetry setup.py test --addopts "${ARGS}"
|
||||
else
|
||||
coverage run --source=moztelemetry setup.py test
|
||||
fi
|
||||
coverage xml
|
|
@ -4,8 +4,10 @@ test=pytest
|
|||
[flake8]
|
||||
max-line-length=100
|
||||
exclude=
|
||||
.svn,CVS,.bzr,.hg,.git,__pycache__,.*tox,.eggs,*.egg,
|
||||
moztelemetry/parse_histograms.py,
|
||||
moztelemetry/heka/message_pb2.py
|
||||
moztelemetry/heka/message_pb2.py,
|
||||
.*tox
|
||||
|
||||
# * E226: allow whitespace around arithmetic operators
|
||||
# * E221,E241,E251: ignore spaces around keyword arguments and dict entries,
|
||||
|
@ -14,7 +16,8 @@ exclude=
|
|||
ignore=E221,E226,E241,E251,E501
|
||||
|
||||
[coverage:run]
|
||||
omit=
|
||||
source = moztelemetry/
|
||||
omit =
|
||||
moztelemetry/shared_telemetry_utils.py,
|
||||
moztelemetry/parse_histograms.py,
|
||||
moztelemetry/heka/message_pb2.py
|
||||
|
|
8
setup.py
8
setup.py
|
@ -6,6 +6,9 @@
|
|||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
tests_require = ['mock', 'pytest-timeout', 'moto', 'responses',
|
||||
'scipy', 'pyspark', 'pytest', 'pytest-cov']
|
||||
|
||||
setup(
|
||||
name='python_moztelemetry',
|
||||
use_scm_version=True,
|
||||
|
@ -21,8 +24,7 @@ setup(
|
|||
'PyYAML', 'python-snappy'],
|
||||
setup_requires=['pytest-runner', 'setuptools_scm'],
|
||||
extras_require={
|
||||
'test': ['mock', 'pytest-timeout', 'moto<=1.1.22', 'responses',
|
||||
'scipy', 'pytest-flake8', 'pytest', 'pytest-cov'],
|
||||
'testing': tests_require,
|
||||
},
|
||||
tests_require=['python_moztelemetry[test]'],
|
||||
tests_require=tests_require,
|
||||
)
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
# Tox (https://tox.readthedocs.io/) is a tool for maintaining
|
||||
# multiple virtualenv environments for different python versions.
|
||||
# This file is referenced when tox is invoked in bin/test or .circleci/config.yml
|
||||
|
||||
[tox]
|
||||
envlist = py36 # CircleCI jobs override the envlist; see .circleci/config.yml
|
||||
|
||||
[pytest]
|
||||
addopts =
|
||||
-rsxX
|
||||
--verbose
|
||||
--showlocals
|
||||
--tb=native
|
||||
--timeout=120
|
||||
--capture=no
|
||||
--cov
|
||||
|
||||
[testenv]
|
||||
skipsdist = True
|
||||
usedevelop = True
|
||||
extras = testing
|
||||
commands = pytest {posargs:tests/}
|
||||
|
||||
[testenv:lint]
|
||||
deps =
|
||||
flake8
|
||||
commands =
|
||||
flake8 moztelemetry/ tests/
|
Загрузка…
Ссылка в новой задаче