bugbug/infra/data-pipeline.yml

version: 1
tasks:
  $let:
    year: {$eval: 'now[0:4]'}
    month: {$eval: 'now[5:7]'}
    day: {$eval: 'now[8:10]'}
    hour: {$eval: 'now[11:13]'}
    minute: {$eval: 'now[14:16]'}
    second: {$eval: 'now[17:19]'}
    repository: https://github.com/mozilla/bugbug
    taskboot_image: mozilla/taskboot:0.1.10
  in:
    - ID: microannotate-generate-tokenize
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: ci
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 604800
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-microannotate-generate
          - /cache/
          - https://github.com/marco-c/gecko-dev-wordified
          - --tokenize
        cache:
          bugbug-mercurial-repository: /cache
        features:
          taskclusterProxy:
            true
      scopes:
        - "docker-worker:cache:bugbug-mercurial-repository"
        - "secrets:get:project/relman/bugbug/production"
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.microannotate_gecko-dev-wordified.${version}
        - index.project.relman.bugbug.microannotate_gecko-dev-wordified.latest
      metadata:
        name: bugbug microannotate tokenized repository generator
        description: bugbug microannotate tokenized repository generator
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: microannotate-generate-remove-comments
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: ci
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 432000
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-microannotate-generate
          - /cache/
          - https://github.com/marco-c/gecko-dev-comments-removed
          - --remove-comments
        cache:
          bugbug-mercurial-repository: /cache
        features:
          taskclusterProxy:
            true
      scopes:
        - "docker-worker:cache:bugbug-mercurial-repository"
        - "secrets:get:project/relman/bugbug/production"
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.microannotate_gecko-dev-comments-removed.${version}
        - index.project.relman.bugbug.microannotate_gecko-dev-comments-removed.latest
      metadata:
        name: bugbug microannotate repository with comments removed generator
        description: bugbug microannotate repository with comments removed generator
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: microannotate-generate-tokenize-and-remove-comments
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: ci
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 432000
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-microannotate-generate
          - /cache/
          - https://github.com/marco-c/gecko-dev-wordified-and-comments-removed
          - --tokenize
          - --remove-comments
        cache:
          bugbug-mercurial-repository: /cache
        features:
          taskclusterProxy:
            true
      scopes:
        - "docker-worker:cache:bugbug-mercurial-repository"
        - "secrets:get:project/relman/bugbug/production"
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.microannotate_gecko-dev-wordified-and-comments-removed.${version}
        - index.project.relman.bugbug.microannotate_gecko-dev-wordified-and-comments-removed.latest
      metadata:
        name: bugbug microannotate tokenized repository with comments removed generator
        description: bugbug microannotate tokenized repository with comments removed generator
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: commit-retrieval
      created: {$fromNow: ''}
      deadline: {$fromNow: '1 day'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-large
      payload:
        maxRunTime: 86400
        image: mozilla/bugbug-commit-retrieval:${version}

        artifacts:
          public/commits.json.zst:
            path: /data/commits.json.zst
            type: file
          public/commits.json.version:
            path: /data/commits.json.version
            type: file
          public/commit_experiences.pickle.zst:
            path: /data/commit_experiences.pickle.zst
            type: file
        cache:
          bugbug-mercurial-repository: /cache
      scopes:
        - "docker-worker:cache:bugbug-mercurial-repository"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.data_commits.${version}
        - index.project.relman.bugbug.data_commits.latest
      metadata:
        name: bugbug commit retrieval
        description: bugbug commit retrieval
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: bugs-retrieval
      created: {$fromNow: ''}
      deadline: {$fromNow: '2 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      dependencies:
        - commit-retrieval
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 86400
        image: mozilla/bugbug-base:${version}
        command:
          - "bugbug-data-bugzilla"

        artifacts:
          public/bugs.json.zst:
            path: /data/bugs.json.zst
            type: file
          public/bugs.json.version:
            path: /data/bugs.json.version
            type: file

        features:
          taskclusterProxy:
            true
      scopes:
        - "secrets:get:project/relman/bugbug/production"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.data_bugs.${version}
        - index.project.relman.bugbug.data_bugs.latest
      metadata:
        name: bugbug bugs retrieval
        description: bugbug bugs retrieval
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: test-scheduling-history-push_data-retrieval
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      dependencies:
        - commit-retrieval
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 259200
        image: ahal/adr
        command:
          - "/bin/bash"
          - "-cx"
          - "wget ${repository}/raw/${version}/scripts/test_scheduling_history_retriever.py &&
             pip install --disable-pip-version-check --quiet --no-cache-dir bugbug==${version} &&
             python test_scheduling_history_retriever.py retrieve"

        artifacts:
          public/push_data_label.json.zst:
            path: /builds/worker/push_data_label.json.zst
            type: file
          public/push_data_group.json.zst:
            path: /builds/worker/push_data_group.json.zst
            type: file

        features:
          taskclusterProxy:
            true
      scopes:
        - "secrets:get:project/relman/bugbug/production"
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.data_test_scheduling_history_push_data.${version}
        - index.project.relman.bugbug.data_test_scheduling_history_push_data.latest
      metadata:
        name: bugbug test scheduling history push data retrieval
        description: bugbug test scheduling history push data retrieval
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: test-label-scheduling-history-generator
      created: {$fromNow: ''}
      deadline: {$fromNow: '4 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - test-scheduling-history-push_data-retrieval
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 86400
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-data-test-scheduling-history
          - generate
          - --granularity=label

        artifacts:
          public/test_label_scheduling_history.pickle.zst:
            path: /data/test_label_scheduling_history.pickle.zst
            type: file
          public/test_label_scheduling_history.pickle.version:
            path: /data/test_label_scheduling_history.pickle.version
            type: file
          public/past_failures_label.lmdb.tar.zst:
            path: /data/past_failures_label.lmdb.tar.zst
            type: file

        features:
          taskclusterProxy:
            true
      scopes:
        - "secrets:get:project/relman/bugbug/production"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.data_test_label_scheduling_history.${version}
        - index.project.relman.bugbug.data_test_label_scheduling_history.latest
      metadata:
        name: bugbug test label scheduling history retrieval
        description: bugbug test label scheduling history retrieval
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: test-group-scheduling-history-generator
      created: {$fromNow: ''}
      deadline: {$fromNow: '4 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - test-scheduling-history-push_data-retrieval
      payload:
        env:
          TC_SECRET_ID: project/relman/bugbug/production
        maxRunTime: 86400
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-data-test-scheduling-history
          - generate
          - --granularity=group

        artifacts:
          public/test_group_scheduling_history.pickle.zst:
            path: /data/test_group_scheduling_history.pickle.zst
            type: file
          public/test_group_scheduling_history.pickle.version:
            path: /data/test_group_scheduling_history.pickle.version
            type: file
          public/past_failures_group.lmdb.tar.zst:
            path: /data/past_failures_group.lmdb.tar.zst
            type: file

        features:
          taskclusterProxy:
            true
      scopes:
        - "secrets:get:project/relman/bugbug/production"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.data_test_group_scheduling_history.${version}
        - index.project.relman.bugbug.data_test_group_scheduling_history.latest
      metadata:
        name: bugbug test group scheduling history retrieval
        description: bugbug test group scheduling history retrieval
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: rollback-test-task
      created: {$fromNow: ''}
      deadline: {$fromNow: '2 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 3600
        image: mozilla/bugbug-base:${version}
        command:
          - "/bin/bash"
          - "-lcx"
          - "python -c 'from bugbug import bugzilla, db; db.download(bugzilla.BUGS_DB)' &&
             python -m bugbug.bug_snapshot --verbose"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
      metadata:
        name: bugbug rollback test
        description: bugbug rollback test
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-backout
      created: {$fromNow: ''}
      deadline: {$fromNow: '2 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - commit-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - backout

        artifacts:
          public/backoutmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /backoutmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_backout.${version}
        - index.project.relman.bugbug.train_backout.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_backout.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_backout.latest
      metadata:
        name: bugbug train backout model
        description: bugbug train backout model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-component
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - component

        artifacts:
          public/componentmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /componentmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_component.${version}
        - index.project.relman.bugbug.train_component.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_component.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_component.latest
      metadata:
        name: bugbug train component model
        description: bugbug train component model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-defectenhancementtask
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - defectenhancementtask

        artifacts:
          public/defectenhancementtaskmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /defectenhancementtaskmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_defectenhancementtask.${version}
        - index.project.relman.bugbug.train_defectenhancementtask.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_defectenhancementtask.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_defectenhancementtask.latest
      metadata:
        name: bugbug train defect/enhancement/task model
        description: bugbug train defect/enhancement/task model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-regression
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-large
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - regression

        artifacts:
          public/regressionmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /regressionmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file
          public/feature_importance.png:
            expires: {$fromNow: '1 year'}
            path: /feature_importance.png
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_regression.${version}
        - index.project.relman.bugbug.train_regression.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_regression.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_regression.latest
      metadata:
        name: bugbug train regression model
        description: bugbug train regression model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: regressor-finder-commits-to-ignore
      created: {$fromNow: ''}
      deadline: {$fromNow: '1 day'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - commit-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-regressor-finder
          - to_ignore
          - --repo_dir=/cache/mozilla-central
        cache:
          bugbug-mercurial-repository: /cache
        features:
          taskclusterProxy:
            true
      scopes:
        - "docker-worker:cache:bugbug-mercurial-repository"
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug_annotate.regressor_finder_commits_to_ignore.latest
      metadata:
        name: bugbug regressor finder (commits to ignore)
        description: bugbug regressor finder (commits to ignore)
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: regressor-finder-bug-fixing-commits
      created: {$fromNow: ''}
      deadline: {$fromNow: '1 day'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - commit-retrieval
        - bugs-retrieval
        - train-defectenhancementtask
        - train-regression
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-regressor-finder
          - bug_fixing
        features:
          taskclusterProxy:
            true
      scopes:
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug_annotate.regressor_finder_bug_fixing_commits.latest
      metadata:
        name: bugbug regressor finder (bug-fixing commits)
        description: bugbug regressor finder (bug-fixing commits)
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: regressor-finder-tokenized-bug-introducing
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-large
      dependencies:
        - commit-retrieval
        - microannotate-generate-tokenize
        - microannotate-generate-remove-comments
        - microannotate-generate-tokenize-and-remove-comments
        - bugs-retrieval
        - regressor-finder-commits-to-ignore
        - regressor-finder-bug-fixing-commits
      payload:
        maxRunTime: 432000
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-regressor-finder
          - bug_introducing
          - --tokenized_git_repo_url=https://github.com/marco-c/gecko-dev-wordified-and-comments-removed
          - --tokenized_git_repo_dir=gecko-dev-wordified-and-comments-removed
        features:
          taskclusterProxy:
            true
      scopes:
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug_annotate.regressor_finder_tokenized_bug_introducing_commits.latest
      metadata:
        name: bugbug regressor finder (tokenized bug-introducing commits)
        description: bugbug regressor finder (tokenized bug-introducing commits)
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: regressor-finder-bug-introducing
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: compute-large
      dependencies:
        - commit-retrieval
        - microannotate-generate-tokenize
        - microannotate-generate-remove-comments
        - microannotate-generate-tokenize-and-remove-comments
        - bugs-retrieval
        - regressor-finder-commits-to-ignore
        - regressor-finder-bug-fixing-commits
      payload:
        maxRunTime: 432000
        image: mozilla/bugbug-commit-retrieval:${version}
        command:
          - bugbug-regressor-finder
          - bug_introducing
          - --git_repo_url=https://github.com/mozilla/gecko-dev
          - --git_repo_dir=gecko-dev
        features:
          taskclusterProxy:
            true
      scopes:
        - "auth:aws-s3:read-write:communitytc-bugbug/*"
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug_annotate.regressor_finder_bug_introducing_commits.latest
      metadata:
        name: bugbug regressor finder (bug-introducing commits)
        description: bugbug regressor finder (bug-introducing commits)
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: past-bugs-by-function
      created: {$fromNow: ''}
      deadline: {$fromNow: '1 day'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      dependencies:
        - commit-retrieval
        - bugs-retrieval
        - regressor-finder-bug-fixing-commits
      payload:
        maxRunTime: 7200
        image: mozilla/bugbug-base:${version}
        command:
          - "bugbug-past-bugs-by-function"

        artifacts:
          public/past_bugs_by_function.pickle.zst:
            path: /data/past_bugs_by_function.pickle.zst
            type: file
      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.past_bugs_by_function.${version}
        - index.project.relman.bugbug.past_bugs_by_function.latest
      metadata:
        name: bugbug past bugs by function
        description: bugbug past bugs by function
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-regressor
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - commit-retrieval
        - regressor-finder-bug-introducing
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - regressor
          - --interpretable

        artifacts:
          public/regressormodel.zst:
            expires: {$fromNow: '1 month'}
            path: /regressormodel.zst
            type: file
          public/regressormodel_data_X.zst:
            expires: {$fromNow: '1 month'}
            path: /regressormodel_data_X.zst
            type: file
          public/regressormodel_data_y.zst:
            expires: {$fromNow: '1 month'}
            path: /regressormodel_data_y.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file
          public/feature_importance.png:
            expires: {$fromNow: '1 year'}
            path: /feature_importance.png
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_regressor.${version}
        - index.project.relman.bugbug.train_regressor.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_regressor.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_regressor.latest
      metadata:
        name: bugbug train regressor model
        description: bugbug train regressor model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-tracking
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - tracking

        artifacts:
          public/trackingmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /trackingmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_tracking.${version}
        - index.project.relman.bugbug.train_tracking.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_tracking.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_tracking.latest
      metadata:
        name: bugbug train tracking model
        description: bugbug train tracking model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-regressionrange
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - regressionrange

        artifacts:
          public/regressionrangemodel.zst:
            expires: {$fromNow: '1 month'}
            path: /regressionrangemodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file
          public/feature_importance.png:
            expires: {$fromNow: '1 year'}
            path: /feature_importance.png
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_regressionrange.${version}
        - index.project.relman.bugbug.train_regressionrange.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_regressionrange.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_regressionrange.latest
      metadata:
        name: bugbug train regressionrange model
        description: bugbug train regressionrange model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-stepstoreproduce
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - stepstoreproduce

        artifacts:
          public/stepstoreproducemodel.zst:
            expires: {$fromNow: '1 month'}
            path: /stepstoreproducemodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file
          public/feature_importance.png:
            expires: {$fromNow: '1 year'}
            path: /feature_importance.png
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_stepstoreproduce.${version}
        - index.project.relman.bugbug.train_stepstoreproduce.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_stepstoreproduce.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_stepstoreproduce.latest
      metadata:
        name: bugbug train stepstoreproduce model
        description: bugbug train stepstoreproduce model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-spambug
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - spambug

        artifacts:
          public/spambugmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /spambugmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file
          public/feature_importance.png:
            expires: {$fromNow: '1 year'}
            path: /feature_importance.png
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_spambug.${version}
        - index.project.relman.bugbug.train_spambug.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_spambug.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_spambug.latest
      metadata:
        name: bugbug train spambug model
        description: bugbug train spambug model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-duplicate
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - duplicate

        artifacts:
          public/duplicatemodel.zst:
            expires: {$fromNow: '1 month'}
            path: /duplicatemodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_duplicate.${version}
        - index.project.relman.bugbug.train_duplicate.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_duplicate.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_duplicate.latest
      metadata:
        name: bugbug train duplicate model
        description: bugbug train duplicate model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-test-select
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-super-large
      dependencies:
        - commit-retrieval
        - test-label-scheduling-history-generator
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - testselect

        artifacts:
          public/testselectmodel.zst:
            expires: {$fromNow: '1 month'}
            path: /testselectmodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_testselect.${version}
        - index.project.relman.bugbug.train_testselect.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_testselect.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_testselect.latest
      metadata:
        name: bugbug train test selection model
        description: bugbug train test selection model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-test-failure
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-large
      dependencies:
        - commit-retrieval
        - test-label-scheduling-history-generator
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base:${version}
        command:
          - bugbug-train
          - testfailure

        artifacts:
          public/testfailuremodel.zst:
            expires: {$fromNow: '1 month'}
            path: /testfailuremodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_testfailure.${version}
        - index.project.relman.bugbug.train_testfailure.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_testfailure.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_testfailure.latest
      metadata:
        name: bugbug train test failure model
        description: bugbug train test failure model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: train-similarity
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 year'}
      provisionerId: proj-relman
      workerType: compute-small
      dependencies:
        - bugs-retrieval
      payload:
        maxRunTime: 25200
        image: mozilla/bugbug-base-nlp:${version}
        command:
          - bugbug-train-similarity
          - --algorithm
          - bm25

        artifacts:
          public/bm25similarity.similaritymodel.zst:
            expires: {$fromNow: '1 month'}
            path: /bm25similarity.similaritymodel.zst
            type: file
          public/metrics.json:
            expires: {$fromNow: '1 year'}
            path: /metrics.json
            type: file

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
        - index.project.relman.bugbug.train_similarity.${version}
        - index.project.relman.bugbug.train_similarity.per_version.${version}.${year}.${month}.${day}.${hour}.${minute}.${second}
        - index.project.relman.bugbug.train_similarity.per_date.${year}.${month}.${day}.${hour}.${minute}.${second}.${version}
        - index.project.relman.bugbug.train_similarity.latest
      metadata:
        name: bugbug train similarity model
        description: bugbug train similarity model
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: check_metrics
      created: {$fromNow: ''}
      deadline: {$fromNow: '5 days'}
      provisionerId: proj-relman
      workerType: batch
      dependencies:
        - train-backout
        - train-component
        - train-defectenhancementtask
        - train-duplicate
        - train-regression
        - train-regressionrange
        - train-regressor
        - train-spambug
        - train-stepstoreproduce
        - train-tracking
        - train-test-select
        - train-test-failure
      payload:
        maxRunTime: 3600
        image: mozilla/bugbug-base:${version}
        command:
          - "/bin/bash"
          - "-lcx"
          - "mkdir /metrics &&
             mkdir /metrics-output &&
             bugbug-check-all-metrics /metrics /metrics-output"
        artifacts:
          public/bugbug:
            expires: {$fromNow: '2 weeks'}
            path: /metrics-output
            type: directory
      metadata:
        name: bugbug check_metrics
        description: bugbug check_metrics
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: docker-build
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: ci
      dependencies:
        - train-defectenhancementtask
        - train-component
        - train-regression
        - train-stepstoreproduce
        - train-spambug
      payload:
        capabilities:
          privileged: true
        maxRunTime: 3600
        image: ${taskboot_image}
        command:
          - "/bin/sh"
          - "-lcxe"
          - "git clone --quiet ${repository} /code &&
             cd /code &&
             git checkout ${version} &&
             taskboot --cache /cache --target /code build-compose --registry=registry.hub.docker.com --write /images --service bugbug-http-service --tag ${version} --tag latest --build-arg BUGBUG_VERSION=${version} &&
             taskboot --cache /cache --target /code build-compose --registry=registry.hub.docker.com --write /images --service bugbug-http-service-bg-worker --tag ${version} --tag latest --build-arg BUGBUG_VERSION=${version}"
        artifacts:
          public/bugbug:
            expires: {$fromNow: '2 weeks'}
            path: /images
            type: directory
      scopes:
        - docker-worker:capability:privileged

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
      metadata:
        name: bugbug docker http build
        description: bugbug docker http build
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    # It's the same task integration_test as in .taskcluster.yml
    - ID: integration-test
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: ci
      dependencies:
        - docker-build
      scopes:
        - secrets:get:project/relman/bugbug/integration
        - "docker-worker:cache:bugbug-mercurial-repository"
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 10800
        image: mozilla/bugbug-commit-retrieval:${version}
        env:
          TC_SECRET_ID: project/relman/bugbug/integration
          CACHE_DIR: "/cache"
        command:
          - "/bin/bash"
          - "-lcx"
          - "apt-get -qq update &&
            apt-get -qq install -y redis-server &&
            git clone --quiet ${repository} &&
            cd bugbug &&
            git -c advice.detachedHead=false checkout ${version} &&
            python -c 'import os; print(os.environ.keys())' &&
            bash ./scripts/integration_test.sh"
        cache:
          bugbug-mercurial-repository: /cache
      metadata:
        name: bugbug integration test
        description: bugbug integration test
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: update_hook_check_pipeline
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      dependencies:
        - integration-test
      scopes:
        - hooks:modify-hook:project-relman/bugbug-checks
        - assume:hook-id:project-relman/bugbug-checks
      provisionerId: proj-relman
      workerType: batch
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 3600
        image: ${taskboot_image}
        command:
          - "/bin/sh"
          - "-lcxe"
          - "git clone --quiet ${repository} &&
             cd bugbug &&
             git -c advice.detachedHead=false checkout ${version} &&
             python infra/set_hook_version.py ${version} infra/taskcluster-hook-check-models-start.json &&
             taskboot --target . build-hook infra/taskcluster-hook-check-models-start.json project-relman bugbug-checks"
      metadata:
        name: bugbug update check hook
        description: bugbug update check hook
        owner: mcastelluccio@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: update_hook_classify_patch
      created: {$fromNow: ''}
      deadline: {$fromNow: '2 days'}
      expires: {$fromNow: '1 month'}
      dependencies:
        - past-bugs-by-function
        - train-regressor
      scopes:
        - hooks:modify-hook:project-relman/bugbug-classify-patch
        - assume:hook-id:project-relman/bugbug-classify-patch
      provisionerId: proj-relman
      workerType: batch
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 3600
        image: ${taskboot_image}
        command:
          - "/bin/sh"
          - "-lcxe"
          - "git clone --quiet ${repository} &&
             cd bugbug &&
             git -c advice.detachedHead=false checkout ${version} &&
             python infra/set_hook_version.py ${version} infra/taskcluster-hook-classify-patch.json &&
             taskboot --target . build-hook infra/taskcluster-hook-classify-patch.json project-relman bugbug-classify-patch"
      metadata:
        name: bugbug update classify patch hook
        description: bugbug update classify patch hook
        owner: mcastelluccio@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: update_hook_test_select
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      dependencies:
        - train-test-failure
        - train-test-select
      scopes:
          - hooks:modify-hook:project-relman/bugbug-test-select
          - assume:hook-id:project-relman/bugbug-test-select
      provisionerId: proj-relman
      workerType: batch
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 3600
        image: ${taskboot_image}
        command:
          - "/bin/sh"
          - "-lcxe"
          - "git clone --quiet ${repository} &&
             cd bugbug &&
             git -c advice.detachedHead=false checkout ${version} &&
             python infra/set_hook_version.py ${version} infra/taskcluster-hook-test-select.json &&
             taskboot --target . build-hook infra/taskcluster-hook-test-select.json project-relman bugbug-test-select"
      metadata:
        name: bugbug update test select hook
        description: bugbug update test select hook
        owner: mcastelluccio@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: docker-push
      dependencies:
        - docker-build
        - integration-test
      scopes:
        - secrets:get:project/relman/bugbug/deploy
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 3600
        image: ${taskboot_image}
        env:
          TASKCLUSTER_SECRET: project/relman/bugbug/deploy
        command:
          - taskboot
          - push-artifact

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
      metadata:
        name: bugbug docker http push
        description: bugbug docker http push
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml

    - ID: http-deploy
      dependencies:
        - docker-build
      scopes:
        - secrets:get:project/relman/bugbug/deploy
      created: {$fromNow: ''}
      deadline: {$fromNow: '3 days'}
      expires: {$fromNow: '1 month'}
      provisionerId: proj-relman
      workerType: batch
      payload:
        features:
          taskclusterProxy:
            true
        maxRunTime: 3600
        image: ${taskboot_image}
        env:
          TASKCLUSTER_SECRET: project/relman/bugbug/deploy
        command:
          - "/bin/sh"
          - "-lcxe"
          - "taskboot deploy-heroku --heroku-app bugbug 'web:public/bugbug/bugbug-http-service.tar' 'worker:public/bugbug/bugbug-http-service-bg-worker.tar'"

      routes:
        - notify.email.release-mgmt-analysis@mozilla.com.on-failed
        - notify.irc-channel.#bugbug.on-failed
      metadata:
        name: bugbug docker http service deploy
        description: bugbug docker http service deploy
        owner: release-mgmt-analysis@mozilla.com
        source: ${repository}/raw/master/data-pipeline.yml