bugbug/infra/data-pipeline.yml

1188 строки
42 KiB
YAML
Исходник Обычный вид История

version: 1
tasks:
$let:
year: {$eval: 'now[0:4]'}
month: {$eval: 'now[5:7]'}
day: {$eval: 'now[8:10]'}
hour: {$eval: 'now[11:13]'}
minute: {$eval: 'now[14:16]'}
second: {$eval: 'now[17:19]'}
repository: https://github.com/mozilla/bugbug
taskboot_image: mozilla/taskboot:0.1.10
in:
- ID: microannotate-generate-tokenize
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: ci
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 604800
image: mozilla/bugbug-commit-retrieval:${version}
command:
- bugbug-microannotate-generate
- /cache/
- https://github.com/marco-c/gecko-dev-wordified
- --tokenize
cache:
bugbug-mercurial-repository: /cache
features:
taskclusterProxy:
true
scopes:
- "docker-worker:cache:bugbug-mercurial-repository"
- "secrets:get:project/relman/bugbug/production"
- "auth:aws-s3:read-write:communitytc-bugbug/*"
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
2019-12-21 16:50:40 +03:00
- index.project.relman.bugbug.microannotate_gecko-dev-wordified.${version}
- index.project.relman.bugbug.microannotate_gecko-dev-wordified.latest
metadata:
name: bugbug microannotate tokenized repository generator
description: bugbug microannotate tokenized repository generator
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: microannotate-generate-remove-comments
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: ci
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 432000
image: mozilla/bugbug-commit-retrieval:${version}
command:
- bugbug-microannotate-generate
- /cache/
- https://github.com/marco-c/gecko-dev-comments-removed
- --remove-comments
cache:
bugbug-mercurial-repository: /cache
features:
taskclusterProxy:
true
scopes:
- "docker-worker:cache:bugbug-mercurial-repository"
- "secrets:get:project/relman/bugbug/production"
- "auth:aws-s3:read-write:communitytc-bugbug/*"
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
2019-12-21 16:50:40 +03:00
- index.project.relman.bugbug.microannotate_gecko-dev-comments-removed.${version}
- index.project.relman.bugbug.microannotate_gecko-dev-comments-removed.latest
metadata:
name: bugbug microannotate repository with comments removed generator
description: bugbug microannotate repository with comments removed generator
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: microannotate-generate-tokenize-and-remove-comments
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: ci
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 432000
image: mozilla/bugbug-commit-retrieval:${version}
command:
- bugbug-microannotate-generate
- /cache/
- https://github.com/marco-c/gecko-dev-wordified-and-comments-removed
- --tokenize
- --remove-comments
cache:
bugbug-mercurial-repository: /cache
features:
taskclusterProxy:
true
scopes:
- "docker-worker:cache:bugbug-mercurial-repository"
- "secrets:get:project/relman/bugbug/production"
- "auth:aws-s3:read-write:communitytc-bugbug/*"
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
2019-12-21 16:50:40 +03:00
- index.project.relman.bugbug.microannotate_gecko-dev-wordified-and-comments-removed.${version}
- index.project.relman.bugbug.microannotate_gecko-dev-wordified-and-comments-removed.latest
metadata:
name: bugbug microannotate tokenized repository with comments removed generator
description: bugbug microannotate tokenized repository with comments removed generator
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: commit-retrieval
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '1 day'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: compute-large
payload:
maxRunTime: 86400
image: mozilla/bugbug-commit-retrieval:${version}
artifacts:
public/commits.json.zst:
path: /data/commits.json.zst
type: file
public/commits.json.version:
path: /data/commits.json.version
type: file
public/commit_experiences.pickle.zst:
path: /data/commit_experiences.pickle.zst
type: file
cache:
bugbug-mercurial-repository: /cache
scopes:
- "docker-worker:cache:bugbug-mercurial-repository"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.data_commits.${version}
- index.project.relman.bugbug.data_commits.latest
metadata:
name: bugbug commit retrieval
description: bugbug commit retrieval
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: bugs-retrieval
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '2 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
dependencies:
- commit-retrieval
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 86400
image: mozilla/bugbug-base:${version}
command:
- "bugbug-data-bugzilla"
artifacts:
public/bugs.json.zst:
path: /data/bugs.json.zst
type: file
public/bugs.json.version:
path: /data/bugs.json.version
type: file
features:
taskclusterProxy:
true
scopes:
- "secrets:get:project/relman/bugbug/production"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.data_bugs.${version}
- index.project.relman.bugbug.data_bugs.latest
metadata:
name: bugbug bugs retrieval
description: bugbug bugs retrieval
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: test-scheduling-history-push_data-retrieval
created: {$fromNow: ''}
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
dependencies:
- commit-retrieval
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 259200
image: ahal/adr
command:
- "/bin/bash"
2019-09-12 15:17:20 +03:00
- "-cx"
- "wget ${repository}/raw/${version}/scripts/test_scheduling_history_retriever.py &&
pip install --disable-pip-version-check --quiet --no-cache-dir bugbug==${version} &&
python test_scheduling_history_retriever.py retrieve"
artifacts:
public/push_data_label.json.zst:
path: /builds/worker/push_data_label.json.zst
type: file
public/push_data_group.json.zst:
path: /builds/worker/push_data_group.json.zst
type: file
features:
taskclusterProxy:
true
scopes:
- "secrets:get:project/relman/bugbug/production"
- "auth:aws-s3:read-write:communitytc-bugbug/*"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.data_test_scheduling_history_push_data.${version}
- index.project.relman.bugbug.data_test_scheduling_history_push_data.latest
metadata:
name: bugbug test scheduling history push data retrieval
description: bugbug test scheduling history push data retrieval
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: test-scheduling-history-generator
created: {$fromNow: ''}
deadline: {$fromNow: '4 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- test-scheduling-history-push_data-retrieval
payload:
env:
TC_SECRET_ID: project/relman/bugbug/production
maxRunTime: 86400
image: mozilla/bugbug-base:${version}
command:
- bugbug-data-test-scheduling-history
- generate
artifacts:
public/test_scheduling_history.pickle.zst:
path: /data/test_scheduling_history.pickle.zst
type: file
public/test_scheduling_history.pickle.version:
path: /data/test_scheduling_history.pickle.version
type: file
public/past_failures.lmdb.tar.zst:
path: /data/past_failures.lmdb.tar.zst
type: file
features:
taskclusterProxy:
true
scopes:
- "secrets:get:project/relman/bugbug/production"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.data_test_scheduling_history.${version}
- index.project.relman.bugbug.data_test_scheduling_history.latest
metadata:
name: bugbug test scheduling history retrieval
description: bugbug test scheduling history retrieval
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: rollback-test-task
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '2 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
dependencies:
- bugs-retrieval
payload:
maxRunTime: 3600
image: mozilla/bugbug-base:${version}
command:
- "/bin/bash"
- "-lcx"
- "python -c 'from bugbug import bugzilla, db; db.download(bugzilla.BUGS_DB)' &&
python -m bugbug.bug_snapshot --verbose"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
metadata:
name: bugbug rollback test
description: bugbug rollback test
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-backout
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '2 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- commit-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- backout
artifacts:
public/backoutmodel.zst:
expires: {$fromNow: '1 month'}
path: /backoutmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_backout.${version}
- index.project.relman.bugbug.train_backout.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_backout.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_backout.latest
metadata:
name: bugbug train backout model
description: bugbug train backout model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-component
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- component
artifacts:
public/componentmodel.zst:
expires: {$fromNow: '1 month'}
path: /componentmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_component.${version}
- index.project.relman.bugbug.train_component.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_component.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_component.latest
metadata:
name: bugbug train component model
description: bugbug train component model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-defectenhancementtask
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- defectenhancementtask
artifacts:
public/defectenhancementtaskmodel.zst:
expires: {$fromNow: '1 month'}
path: /defectenhancementtaskmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_defectenhancementtask.${version}
- index.project.relman.bugbug.train_defectenhancementtask.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_defectenhancementtask.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_defectenhancementtask.latest
metadata:
name: bugbug train defect/enhancement/task model
description: bugbug train defect/enhancement/task model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-regression
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-large
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- regression
artifacts:
public/regressionmodel.zst:
expires: {$fromNow: '1 month'}
path: /regressionmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
public/feature_importance.png:
expires: {$fromNow: '1 year'}
path: /feature_importance.png
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_regression.${version}
- index.project.relman.bugbug.train_regression.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_regression.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_regression.latest
metadata:
name: bugbug train regression model
description: bugbug train regression model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: regressor-finder
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: compute-large
dependencies:
- commit-retrieval
- microannotate-generate-tokenize
- microannotate-generate-remove-comments
- microannotate-generate-tokenize-and-remove-comments
- bugs-retrieval
- train-defectenhancementtask
- train-regression
payload:
maxRunTime: 432000
image: mozilla/bugbug-commit-retrieval:${version}
command:
- bugbug-regressor-finder
- /cache/
- https://github.com/mozilla/gecko-dev
- gecko-dev
- https://github.com/marco-c/gecko-dev-wordified-and-comments-removed
- gecko-dev-wordified-and-comments-removed
cache:
bugbug-mercurial-repository: /cache
features:
taskclusterProxy:
true
scopes:
- "docker-worker:cache:bugbug-mercurial-repository"
- "auth:aws-s3:read-write:communitytc-bugbug/*"
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug_annotate.regressor_finder.latest
metadata:
name: bugbug regressor finder
description: bugbug regressor finder
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: past-bugs-by-function
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
dependencies:
- commit-retrieval
- bugs-retrieval
- regressor-finder
payload:
maxRunTime: 7200
image: mozilla/bugbug-base:${version}
command:
- "bugbug-past-bugs-by-function"
artifacts:
public/past_bugs_by_function.pickle.zst:
path: /data/past_bugs_by_function.pickle.zst
type: file
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.past_bugs_by_function.${version}
- index.project.relman.bugbug.past_bugs_by_function.latest
metadata:
name: bugbug past bugs by function
description: bugbug past bugs by function
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-regressor
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- commit-retrieval
- regressor-finder
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- regressor
- --interpretable
artifacts:
public/regressormodel.zst:
expires: {$fromNow: '1 month'}
path: /regressormodel.zst
type: file
public/regressormodel_data_X.zst:
expires: {$fromNow: '1 month'}
path: /regressormodel_data_X.zst
type: file
public/regressormodel_data_y.zst:
expires: {$fromNow: '1 month'}
path: /regressormodel_data_y.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
public/feature_importance.png:
expires: {$fromNow: '1 year'}
path: /feature_importance.png
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_regressor.${version}
- index.project.relman.bugbug.train_regressor.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_regressor.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_regressor.latest
metadata:
name: bugbug train regressor model
description: bugbug train regressor model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-tracking
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- tracking
artifacts:
public/trackingmodel.zst:
expires: {$fromNow: '1 month'}
path: /trackingmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_tracking.${version}
- index.project.relman.bugbug.train_tracking.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_tracking.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_tracking.latest
metadata:
name: bugbug train tracking model
description: bugbug train tracking model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-regressionrange
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- regressionrange
artifacts:
public/regressionrangemodel.zst:
expires: {$fromNow: '1 month'}
path: /regressionrangemodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
public/feature_importance.png:
expires: {$fromNow: '1 year'}
path: /feature_importance.png
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_regressionrange.${version}
- index.project.relman.bugbug.train_regressionrange.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_regressionrange.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_regressionrange.latest
metadata:
name: bugbug train regressionrange model
description: bugbug train regressionrange model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-stepstoreproduce
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- stepstoreproduce
artifacts:
public/stepstoreproducemodel.zst:
expires: {$fromNow: '1 month'}
path: /stepstoreproducemodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
public/feature_importance.png:
expires: {$fromNow: '1 year'}
path: /feature_importance.png
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_stepstoreproduce.${version}
- index.project.relman.bugbug.train_stepstoreproduce.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_stepstoreproduce.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_stepstoreproduce.latest
metadata:
name: bugbug train stepstoreproduce model
description: bugbug train stepstoreproduce model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-spambug
created: {$fromNow: ''}
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
command:
- bugbug-train
- spambug
artifacts:
public/spambugmodel.zst:
expires: {$fromNow: '1 month'}
path: /spambugmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
public/feature_importance.png:
expires: {$fromNow: '1 year'}
path: /feature_importance.png
type: file
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_spambug.${version}
- index.project.relman.bugbug.train_spambug.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_spambug.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_spambug.latest
metadata:
name: bugbug train spambug model
description: bugbug train spambug model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-duplicate
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
2019-07-02 14:26:46 +03:00
command:
- bugbug-train
- duplicate
artifacts:
public/duplicatemodel.zst:
expires: {$fromNow: '1 month'}
path: /duplicatemodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_duplicate.${version}
- index.project.relman.bugbug.train_duplicate.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_duplicate.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_duplicate.latest
metadata:
name: bugbug train duplicate model
description: bugbug train duplicate model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-test-select
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-super-large
dependencies:
- commit-retrieval
- test-scheduling-history-generator
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
command:
- bugbug-train
- testselect
artifacts:
public/testselectmodel.zst:
expires: {$fromNow: '1 month'}
path: /testselectmodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_testselect.${version}
- index.project.relman.bugbug.train_testselect.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_testselect.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_testselect.latest
metadata:
name: bugbug train test selection model
description: bugbug train test selection model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-test-failure
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-large
dependencies:
- commit-retrieval
- test-scheduling-history-generator
payload:
maxRunTime: 25200
image: mozilla/bugbug-base:${version}
command:
- bugbug-train
- testfailure
artifacts:
public/testfailuremodel.zst:
expires: {$fromNow: '1 month'}
path: /testfailuremodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_testfailure.${version}
- index.project.relman.bugbug.train_testfailure.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_testfailure.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_testfailure.latest
metadata:
name: bugbug train test failure model
description: bugbug train test failure model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: train-similarity
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 year'}
provisionerId: proj-relman
workerType: compute-small
dependencies:
- bugs-retrieval
payload:
maxRunTime: 25200
image: mozilla/bugbug-base-nlp:${version}
command:
- bugbug-train-similarity
- --algorithm
- bm25
artifacts:
public/bm25similarity.similaritymodel.zst:
expires: {$fromNow: '1 month'}
path: /bm25similarity.similaritymodel.zst
type: file
public/metrics.json:
expires: {$fromNow: '1 year'}
path: /metrics.json
type: file
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
- index.project.relman.bugbug.train_similarity.${version}
- index.project.relman.bugbug.train_similarity.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
- index.project.relman.bugbug.train_similarity.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
- index.project.relman.bugbug.train_similarity.latest
metadata:
name: bugbug train similarity model
description: bugbug train similarity model
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: check_metrics
created: {$fromNow: ''}
deadline: {$fromNow: '5 days'}
provisionerId: proj-relman
workerType: batch
dependencies:
- train-backout
- train-component
- train-defectenhancementtask
- train-duplicate
- train-regression
- train-regressionrange
- train-regressor
- train-spambug
- train-stepstoreproduce
- train-tracking
- train-test-select
- train-test-failure
payload:
maxRunTime: 3600
image: mozilla/bugbug-base:${version}
command:
- "/bin/bash"
- "-lcx"
- "mkdir /metrics &&
mkdir /metrics-output &&
bugbug-check-all-metrics /metrics /metrics-output"
artifacts:
public/bugbug:
expires: {$fromNow: '2 weeks'}
path: /metrics-output
type: directory
metadata:
name: bugbug check_metrics
description: bugbug check_metrics
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: docker-build
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: ci
dependencies:
- train-defectenhancementtask
- train-component
- train-regression
- train-stepstoreproduce
- train-spambug
payload:
capabilities:
privileged: true
maxRunTime: 3600
image: ${taskboot_image}
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} /code &&
cd /code &&
git checkout ${version} &&
taskboot --cache /cache --target /code build-compose --registry=registry.hub.docker.com --write /images --service bugbug-http-service --tag ${version} --tag latest --build-arg BUGBUG_VERSION=${version} &&
taskboot --cache /cache --target /code build-compose --registry=registry.hub.docker.com --write /images --service bugbug-http-service-bg-worker --tag ${version} --tag latest --build-arg BUGBUG_VERSION=${version}"
artifacts:
public/bugbug:
expires: {$fromNow: '2 weeks'}
path: /images
type: directory
scopes:
- docker-worker:capability:privileged
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
metadata:
name: bugbug docker http build
description: bugbug docker http build
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
# It's the same task integration_test as in .taskcluster.yml
- ID: integration-test
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: ci
dependencies:
- docker-build
scopes:
- secrets:get:project/relman/bugbug/integration
- "docker-worker:cache:bugbug-mercurial-repository"
payload:
features:
taskclusterProxy:
true
maxRunTime: 10800
image: mozilla/bugbug-commit-retrieval:${version}
env:
TC_SECRET_ID: project/relman/bugbug/integration
CACHE_DIR: "/cache"
command:
- "/bin/bash"
- "-lcx"
- "apt-get -qq update &&
apt-get -qq install -y redis-server &&
git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${version} &&
python -c 'import os; print(os.environ.keys())' &&
bash ./scripts/integration_test.sh"
cache:
bugbug-mercurial-repository: /cache
metadata:
name: bugbug integration test
description: bugbug integration test
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: update_hook_check_pipeline
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
dependencies:
- integration-test
scopes:
- hooks:modify-hook:project-relman/bugbug-checks
- assume:hook-id:project-relman/bugbug-checks
provisionerId: proj-relman
workerType: batch
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: ${taskboot_image}
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${version} &&
python infra/set_hook_version.py ${version} infra/taskcluster-hook-check-models-start.json &&
taskboot --target . build-hook infra/taskcluster-hook-check-models-start.json project-relman bugbug-checks"
metadata:
name: bugbug update check hook
description: bugbug update check hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: update_hook_classify_patch
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '2 days'}
expires: {$fromNow: '1 month'}
dependencies:
- past-bugs-by-function
- train-regressor
scopes:
- hooks:modify-hook:project-relman/bugbug-classify-patch
- assume:hook-id:project-relman/bugbug-classify-patch
provisionerId: proj-relman
workerType: batch
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: ${taskboot_image}
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${version} &&
python infra/set_hook_version.py ${version} infra/taskcluster-hook-classify-patch.json &&
taskboot --target . build-hook infra/taskcluster-hook-classify-patch.json project-relman bugbug-classify-patch"
metadata:
name: bugbug update classify patch hook
description: bugbug update classify patch hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: update_hook_test_select
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
dependencies:
- train-test-failure
- train-test-select
scopes:
- hooks:modify-hook:project-relman/bugbug-test-select
- assume:hook-id:project-relman/bugbug-test-select
provisionerId: proj-relman
workerType: batch
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: ${taskboot_image}
command:
- "/bin/sh"
- "-lcxe"
- "git clone --quiet ${repository} &&
cd bugbug &&
git -c advice.detachedHead=false checkout ${version} &&
python infra/set_hook_version.py ${version} infra/taskcluster-hook-test-select.json &&
taskboot --target . build-hook infra/taskcluster-hook-test-select.json project-relman bugbug-test-select"
metadata:
name: bugbug update test select hook
description: bugbug update test select hook
owner: mcastelluccio@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: docker-push
dependencies:
- docker-build
- integration-test
scopes:
- secrets:get:project/relman/bugbug/deploy
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: ${taskboot_image}
env:
TASKCLUSTER_SECRET: project/relman/bugbug/deploy
command:
- taskboot
- push-artifact
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
metadata:
name: bugbug docker http push
description: bugbug docker http push
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml
- ID: http-deploy
dependencies:
- docker-build
scopes:
- secrets:get:project/relman/bugbug/deploy
created: {$fromNow: ''}
2019-12-16 18:32:46 +03:00
deadline: {$fromNow: '3 days'}
expires: {$fromNow: '1 month'}
provisionerId: proj-relman
workerType: batch
payload:
features:
taskclusterProxy:
true
maxRunTime: 3600
image: ${taskboot_image}
env:
TASKCLUSTER_SECRET: project/relman/bugbug/deploy
command:
- "/bin/sh"
- "-lcxe"
- "taskboot deploy-heroku --heroku-app bugbug 'web:public/bugbug/bugbug-http-service.tar' 'worker:public/bugbug/bugbug-http-service-bg-worker.tar'"
routes:
2019-11-11 23:48:18 +03:00
- notify.email.release-mgmt-analysis@mozilla.com.on-failed
- notify.irc-channel.#bugbug.on-failed
metadata:
name: bugbug docker http service deploy
description: bugbug docker http service deploy
owner: release-mgmt-analysis@mozilla.com
source: ${repository}/raw/master/data-pipeline.yml