bugbug/.taskcluster.yml

450 строки
18 KiB
YAML

version: 1
policy:
pullRequests: public
tasks:
$let:
user: ${event.sender.login}
head_branch:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.ref}
else:
$if: 'tasks_for == "github-push"'
then: ${event.ref}
else: ${event.release.target_commitish}
head_rev:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.sha}
else:
$if: 'tasks_for == "github-push"'
then: ${event.after}
else: ${event.release.tag_name}
repository:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.repo.html_url}
else: ${event.repository.html_url}
taskboot_image: "mozilla/taskboot:0.1.10"
in:
$if: 'tasks_for in ["github-pull-request", "github-push"]'
then:
- taskId: {$eval: as_slugid("lint_task")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
maxRunTime: 3600
image: python:3.7
command:
- "/bin/bash"
- "-lcx"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
pre-commit run -a"
metadata:
name: bugbug lint
description: bugbug lint
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push"'
then:
taskId: {$eval: as_slugid("version_check_task")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
maxRunTime: 3600
image: python:3.7
command:
- "/bin/bash"
- "-lcx"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
python infra/version_check.py"
metadata:
name: bugbug tag version check
description: bugbug tag version check
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- taskId: {$eval: as_slugid("tests_task")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
maxRunTime: 3600
image: python:3.7
env:
CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
command:
- "/bin/bash"
- "-lcx"
- "apt-get -qq update &&
apt-get -qq install -y python-pip libhdf5-dev &&
python2 -m pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==5.1 &&
git clone --quiet https://github.com/mozilla/version-control-tools.git &&
git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
cp infra/hgrc /root/.hgrc &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r requirements.txt &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r extra-nlp-requirements.txt &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r extra-nn-requirements.txt &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r infra/spawn_pipeline_requirements.txt &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
python -m pytest --cov=./ tests/test_*.py &&
bash <(curl -s https://codecov.io/bash)"
metadata:
name: bugbug tests
description: bugbug tests
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- taskId: {$eval: as_slugid("http_tests_task")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
maxRunTime: 3600
image: python:3.7
env:
CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
command:
- "/bin/bash"
- "-lcx"
- "apt-get -qq update &&
apt-get -qq install -y python-pip &&
git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
pip install --disable-pip-version-check --quiet --no-cache-dir . &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r http_service/requirements.txt &&
pytest --cov=http_service http_service/tests/ -vvv &&
bash <(curl -s https://codecov.io/bash)"
metadata:
name: bugbug http service tests
description: bugbug http service tests
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- taskId: {$eval: as_slugid("packaging_test_task")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
maxRunTime: 3600
image: python:3.7
env:
CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
command:
- "/bin/bash"
- "-lcx"
- "apt-get -qq update &&
apt-get -qq install -y libhdf5-dev &&
git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
python -m coverage run setup.py sdist &&
pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz &&
pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz[nlp] &&
pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz[nn] &&
bash <(curl -s https://codecov.io/bash)"
metadata:
name: bugbug packaging test
description: bugbug packaging test
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- taskId: {$eval: as_slugid("docker_build")}
created: {$fromNow: ''}
deadline: {$fromNow: '3 hours'}
provisionerId: proj-relman
workerType: ci
payload:
capabilities:
privileged: true
maxRunTime: 10800
image: "${taskboot_image}"
env:
REGISTRY: registry.hub.docker.com
VERSION:
$if: 'head_branch[:10] == "refs/tags/"'
then: {$eval: 'head_branch[10:]' }
else: "latest"
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} /code &&
cd /code &&
git -c advice.detachedHead=false checkout ${head_rev} &&
taskboot --cache /cache --target /code build-compose --write /images --build-arg CHECK_MODELS=0 --tag $VERSION --tag latest"
artifacts:
public/bugbug:
expires: {$fromNow: '2 weeks'}
path: /images
type: directory
cache:
bugbug-build: /cache
scopes:
- docker-worker:capability:privileged
- docker-worker:cache:bugbug-build
metadata:
name: bugbug docker build
description: bugbug docker build
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
dependencies:
- {$eval: as_slugid("lint_task")}
- {$eval: as_slugid("tests_task")}
- {$eval: as_slugid("http_tests_task")}
- {$eval: as_slugid("packaging_test_task")}
- {$eval: as_slugid("version_check_task")}
- {$eval: as_slugid("integration_test")}
scopes:
- secrets:get:project/relman/bugbug/deploy
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
command:
- taskboot
- deploy-pypi
env:
TASKCLUSTER_SECRET: project/relman/bugbug/deploy
GIT_REPOSITORY: "${repository}"
GIT_REVISION: "${head_rev}"
metadata:
name: bugbug PyPI release
description: bugbug PyPI release
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
dependencies:
- {$eval: as_slugid("docker_build")}
- {$eval: as_slugid("lint_task")}
- {$eval: as_slugid("version_check_task")}
- {$eval: as_slugid("tests_task")}
- {$eval: as_slugid("packaging_test_task")}
- {$eval: as_slugid("integration_test")}
scopes:
- secrets:get:project/relman/bugbug/deploy
taskId: {$eval: as_slugid("docker_push")}
created: {$fromNow: ''}
deadline: {$fromNow: '4 hours'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
env:
TASKCLUSTER_SECRET: project/relman/bugbug/deploy
command:
- taskboot
- push-artifact
- --exclude-filter
- "*http-service*"
metadata:
name: bugbug docker push
description: bugbug docker push
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
taskId: {$eval: as_slugid("update_hook_annotate_pipeline")}
dependencies:
- {$eval: as_slugid("docker_push")}
scopes:
- hooks:modify-hook:project-relman/bugbug-annotate
- assume:hook-id:project-relman/bugbug-annotate
created: {$fromNow: ''}
deadline: {$fromNow: '5 hours'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
env:
VERSION: {$eval: 'head_branch[10:]'}
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
python infra/set_hook_version.py $VERSION infra/taskcluster-hook-annotate.json &&
taskboot --target . build-hook infra/taskcluster-hook-annotate.json project-relman bugbug-annotate"
metadata:
name: bugbug update annotate hook
description: bugbug update annotate hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
taskId: {$eval: as_slugid("update_hook_data_pipeline")}
dependencies:
- {$eval: as_slugid("docker_push")}
scopes:
- hooks:modify-hook:project-relman/bugbug
- assume:hook-id:project-relman/bugbug
- queue:route:project.relman.bugbug.deploy_ending.*
created: {$fromNow: ''}
deadline: {$fromNow: '5 hours'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-pipeline-start.json &&
taskboot --target . build-hook infra/taskcluster-hook-pipeline-start.json project-relman bugbug"
routes:
- project.relman.bugbug.deploy_ending
metadata:
name: bugbug update data hook
description: bugbug update data hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
taskId: {$eval: as_slugid("update_hook_check_pipeline")}
dependencies:
- {$eval: as_slugid("docker_push")}
scopes:
- hooks:modify-hook:project-relman/bugbug-checks
- assume:hook-id:project-relman/bugbug-checks
created: {$fromNow: ''}
deadline: {$fromNow: '5 hours'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-check-models-start.json &&
taskboot --target . build-hook infra/taskcluster-hook-check-models-start.json project-relman bugbug-checks"
metadata:
name: bugbug update check hook
description: bugbug update check hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
taskId: {$eval: as_slugid("update_hook_classify_patch")}
dependencies:
- {$eval: as_slugid("docker_push")}
scopes:
- hooks:modify-hook:project-relman/bugbug-classify-patch
- assume:hook-id:project-relman/bugbug-classify-patch
created: {$fromNow: ''}
deadline: {$fromNow: '5 hours'}
provisionerId: proj-relman
workerType: ci
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: "${taskboot_image}"
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-classify-patch.json &&
taskboot --target . build-hook infra/taskcluster-hook-classify-patch.json project-relman bugbug-classify-patch"
metadata:
name: bugbug update classify patch hook
description: bugbug update classify patch hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
# It's the same task integration_test as in data-pipeline.yml
- $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
then:
taskId: {$eval: as_slugid("integration_test")}
created: {$fromNow: ''}
deadline: {$fromNow: '1 hour'}
provisionerId: proj-relman
workerType: ci
scopes:
- secrets:get:project/relman/bugbug/integration
- "docker-worker:cache:bugbug-mercurial-repository"
payload:
features:
taskclusterProxy:
true
maxRunTime: 10800
image: mozilla/bugbug-commit-retrieval:latest
env:
TC_SECRET_ID: project/relman/bugbug/integration
CACHE_DIR: "/cache"
command:
- "/bin/bash"
- "-lcx"
- "apt-get -qq update &&
apt-get -qq install -y python-pip redis-server &&
git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${head_rev} &&
pip install --disable-pip-version-check --quiet --no-cache-dir . &&
python -c 'import os; print(os.environ.keys())' &&
bash ./scripts/integration_test.sh"
cache:
bugbug-mercurial-repository: /cache
metadata:
name: bugbug integration test
description: bugbug integration test
owner: ${user}@users.noreply.github.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml