bugbug/.taskcluster.yml

version: 1
policy:
  pullRequests: public
tasks:
  $let:
    user: ${event.sender.login}

    head_branch:
      $if: 'tasks_for == "github-pull-request"'
      then: ${event.pull_request.head.ref}
      else:
        $if: 'tasks_for == "github-push"'
        then: ${event.ref}
        else: ${event.release.target_commitish}

    head_rev:
      $if: 'tasks_for == "github-pull-request"'
      then: ${event.pull_request.head.sha}
      else:
        $if: 'tasks_for == "github-push"'
        then: ${event.after}
        else: ${event.release.tag_name}

    repository:
      $if: 'tasks_for == "github-pull-request"'
      then: ${event.pull_request.head.repo.html_url}
      else: ${event.repository.html_url}

    taskboot_image: "mozilla/taskboot:0.1.10"
  in:
    $if: 'tasks_for in ["github-pull-request", "github-push"]'
    then:
      - taskId: {$eval: as_slugid("lint_task")}
        created: {$fromNow: ''}
        deadline: {$fromNow: '1 hour'}
        provisionerId: proj-relman
        workerType: ci
        payload:
          maxRunTime: 3600
          image: python:3.7
          command:
            - "/bin/bash"
            - "-lcx"
            - "git clone --quiet ${repository} &&
               cd bugbug &&
               git -c advice.detachedHead=false checkout ${head_rev} &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
               pre-commit run -a"
        metadata:
          name: bugbug lint
          description: bugbug lint
          owner: mcastelluccio@mozilla.com
          source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push"'
        then:
          taskId: {$eval: as_slugid("version_check_task")}
          created: {$fromNow: ''}
          deadline: {$fromNow: '1 hour'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            maxRunTime: 3600
            image: python:3.7
            command:
              - "/bin/bash"
              - "-lcx"
              - "git clone --quiet ${repository} &&
                 cd bugbug &&
                 git -c advice.detachedHead=false checkout ${head_rev} &&
                 python infra/version_check.py"
          metadata:
            name: bugbug tag version check
            description: bugbug tag version check
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - taskId: {$eval: as_slugid("tests_task")}
        created: {$fromNow: ''}
        deadline: {$fromNow: '1 hour'}
        provisionerId: proj-relman
        workerType: ci
        payload:
          maxRunTime: 3600
          image: python:3.7
          env:
            CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
          command:
            - "/bin/bash"
            - "-lcx"
            - "apt-get -qq update &&
               apt-get -qq install -y python-pip libhdf5-dev &&
               python2 -m pip install --disable-pip-version-check --quiet --no-cache-dir mercurial==5.1 &&
               git clone --quiet https://github.com/mozilla/version-control-tools.git &&
               git clone --quiet ${repository} &&
               cd bugbug &&
               git -c advice.detachedHead=false checkout ${head_rev} &&
               cp infra/hgrc /root/.hgrc &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r requirements.txt &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r extra-nlp-requirements.txt &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r extra-nn-requirements.txt &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r infra/spawn_pipeline_requirements.txt &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
               python -m pytest --cov=./ tests/test_*.py &&
               bash <(curl -s https://codecov.io/bash)"
        metadata:
          name: bugbug tests
          description: bugbug tests
          owner: mcastelluccio@mozilla.com
          source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - taskId: {$eval: as_slugid("http_tests_task")}
        created: {$fromNow: ''}
        deadline: {$fromNow: '1 hour'}
        provisionerId: proj-relman
        workerType: ci
        payload:
          maxRunTime: 3600
          image: python:3.7
          env:
            CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
          command:
            - "/bin/bash"
            - "-lcx"
            - "apt-get -qq update &&
               apt-get -qq install -y python-pip &&
               git clone --quiet ${repository} &&
               cd bugbug &&
               git -c advice.detachedHead=false checkout ${head_rev} &&
               pip install --disable-pip-version-check --quiet --no-cache-dir . &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r http_service/requirements.txt &&
               pytest --cov=http_service http_service/tests/ -vvv &&
               bash <(curl -s https://codecov.io/bash)"
        metadata:
          name: bugbug http service tests
          description: bugbug http service tests
          owner: mcastelluccio@mozilla.com
          source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - taskId: {$eval: as_slugid("packaging_test_task")}
        created: {$fromNow: ''}
        deadline: {$fromNow: '1 hour'}
        provisionerId: proj-relman
        workerType: ci
        payload:
          maxRunTime: 3600
          image: python:3.7
          env:
            CODECOV_TOKEN: 66162f89-a4d9-420c-84bd-d10f12a428d9
          command:
            - "/bin/bash"
            - "-lcx"
            - "apt-get -qq update &&
               apt-get -qq install -y libhdf5-dev &&
               git clone --quiet ${repository} &&
               cd bugbug &&
               git -c advice.detachedHead=false checkout ${head_rev} &&
               pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
               python -m coverage run setup.py sdist &&
               pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz &&
               pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz[nlp] &&
               pip install --disable-pip-version-check --quiet --no-cache-dir dist/bugbug-$(cat VERSION).tar.gz[nn] &&
               bash <(curl -s https://codecov.io/bash)"
        metadata:
          name: bugbug packaging test
          description: bugbug packaging test
          owner: mcastelluccio@mozilla.com
          source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - taskId: {$eval: as_slugid("docker_build")}
        created: {$fromNow: ''}
        deadline: {$fromNow: '3 hours'}
        provisionerId: proj-relman
        workerType: ci
        payload:
          capabilities:
            privileged: true
          maxRunTime: 10800
          image: "${taskboot_image}"
          env:
            REGISTRY: registry.hub.docker.com
            VERSION:
              $if: 'head_branch[:10] == "refs/tags/"'
              then: {$eval: 'head_branch[10:]' }
              else: "latest"
          command:
            - "/bin/sh"
            - "-lcxe"
            - "git clone --quiet ${repository} /code &&
               cd /code &&
               git -c advice.detachedHead=false checkout ${head_rev} &&
               taskboot --cache /cache --target /code build-compose --write /images --build-arg CHECK_MODELS=0 --tag $VERSION --tag latest"
          artifacts:
            public/bugbug:
              expires: {$fromNow: '2 weeks'}
              path: /images
              type: directory
          cache:
            bugbug-build: /cache
        scopes:
          - docker-worker:capability:privileged
          - docker-worker:cache:bugbug-build
        metadata:
          name: bugbug docker build
          description: bugbug docker build
          owner: mcastelluccio@mozilla.com
          source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          dependencies:
            - {$eval: as_slugid("lint_task")}
            - {$eval: as_slugid("tests_task")}
            - {$eval: as_slugid("http_tests_task")}
            - {$eval: as_slugid("packaging_test_task")}
            - {$eval: as_slugid("version_check_task")}
            - {$eval: as_slugid("integration_test")}
          scopes:
            - secrets:get:project/relman/bugbug/deploy
          created: {$fromNow: ''}
          deadline: {$fromNow: '1 hour'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            command:
              - taskboot
              - deploy-pypi
            env:
              TASKCLUSTER_SECRET: project/relman/bugbug/deploy
              GIT_REPOSITORY: "${repository}"
              GIT_REVISION: "${head_rev}"
          metadata:
            name: bugbug PyPI release
            description: bugbug PyPI release
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          dependencies:
            - {$eval: as_slugid("docker_build")}
            - {$eval: as_slugid("lint_task")}
            - {$eval: as_slugid("version_check_task")}
            - {$eval: as_slugid("tests_task")}
            - {$eval: as_slugid("packaging_test_task")}
            - {$eval: as_slugid("integration_test")}
          scopes:
            - secrets:get:project/relman/bugbug/deploy
          taskId: {$eval: as_slugid("docker_push")}
          created: {$fromNow: ''}
          deadline: {$fromNow: '4 hours'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            env:
              TASKCLUSTER_SECRET: project/relman/bugbug/deploy
            command:
              - taskboot
              - push-artifact
              - --exclude-filter
              - "*http-service*"
          metadata:
            name: bugbug docker push
            description: bugbug docker push
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          taskId: {$eval: as_slugid("update_hook_annotate_pipeline")}
          dependencies:
            - {$eval: as_slugid("docker_push")}
          scopes:
            - hooks:modify-hook:project-relman/bugbug-annotate
            - assume:hook-id:project-relman/bugbug-annotate
          created: {$fromNow: ''}
          deadline: {$fromNow: '5 hours'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            env:
              VERSION: {$eval: 'head_branch[10:]'}
            command:
              - "/bin/sh"
              - "-lcxe"
              - "git clone --quiet ${repository} &&
                 cd bugbug &&
                 git -c advice.detachedHead=false checkout ${head_rev} &&
                 python infra/set_hook_version.py $VERSION infra/taskcluster-hook-annotate.json &&
                 taskboot --target . build-hook infra/taskcluster-hook-annotate.json project-relman bugbug-annotate"
          metadata:
            name: bugbug update annotate hook
            description: bugbug update annotate hook
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          taskId: {$eval: as_slugid("update_hook_data_pipeline")}
          dependencies:
            - {$eval: as_slugid("docker_push")}
          scopes:
            - hooks:modify-hook:project-relman/bugbug
            - assume:hook-id:project-relman/bugbug
            - queue:route:project.relman.bugbug.deploy_ending.*
          created: {$fromNow: ''}
          deadline: {$fromNow: '5 hours'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            command:
              - "/bin/sh"
              - "-lcxe"
              - "git clone --quiet ${repository} &&
                 cd bugbug &&
                 git -c advice.detachedHead=false checkout ${head_rev} &&
                 python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-pipeline-start.json &&
                 taskboot --target . build-hook infra/taskcluster-hook-pipeline-start.json project-relman bugbug"
          routes:
            - project.relman.bugbug.deploy_ending
          metadata:
            name: bugbug update data hook
            description: bugbug update data hook
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          taskId: {$eval: as_slugid("update_hook_check_pipeline")}
          dependencies:
            - {$eval: as_slugid("docker_push")}
          scopes:
            - hooks:modify-hook:project-relman/bugbug-checks
            - assume:hook-id:project-relman/bugbug-checks
          created: {$fromNow: ''}
          deadline: {$fromNow: '5 hours'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            command:
              - "/bin/sh"
              - "-lcxe"
              - "git clone --quiet ${repository} &&
                 cd bugbug &&
                 git -c advice.detachedHead=false checkout ${head_rev} &&
                 python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-check-models-start.json &&
                 taskboot --target . build-hook infra/taskcluster-hook-check-models-start.json project-relman bugbug-checks"
          metadata:
            name: bugbug update check hook
            description: bugbug update check hook
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          taskId: {$eval: as_slugid("update_hook_classify_patch")}
          dependencies:
            - {$eval: as_slugid("docker_push")}
          scopes:
            - hooks:modify-hook:project-relman/bugbug-classify-patch
            - assume:hook-id:project-relman/bugbug-classify-patch
          created: {$fromNow: ''}
          deadline: {$fromNow: '5 hours'}
          provisionerId: proj-relman
          workerType: ci
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 3600
            image: "${taskboot_image}"
            command:
              - "/bin/sh"
              - "-lcxe"
              - "git clone --quiet ${repository} &&
                 cd bugbug &&
                 git -c advice.detachedHead=false checkout ${head_rev} &&
                 python infra/set_hook_version.py ${head_branch[10:]} infra/taskcluster-hook-classify-patch.json &&
                 taskboot --target . build-hook infra/taskcluster-hook-classify-patch.json project-relman bugbug-classify-patch"
          metadata:
            name: bugbug update classify patch hook
            description: bugbug update classify patch hook
            owner: mcastelluccio@mozilla.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml

      # It's the same task integration_test as in data-pipeline.yml
      - $if: 'tasks_for == "github-push" && head_branch[:10] == "refs/tags/"'
        then:
          taskId: {$eval: as_slugid("integration_test")}
          created: {$fromNow: ''}
          deadline: {$fromNow: '1 hour'}
          provisionerId: proj-relman
          workerType: ci
          scopes:
            - secrets:get:project/relman/bugbug/integration
            - "docker-worker:cache:bugbug-mercurial-repository"
          payload:
            features:
              taskclusterProxy:
                true
            maxRunTime: 10800
            image: mozilla/bugbug-commit-retrieval:latest
            env:
              TC_SECRET_ID: project/relman/bugbug/integration
              CACHE_DIR: "/cache"
            command:
              - "/bin/bash"
              - "-lcx"
              - "apt-get -qq update &&
                apt-get -qq install -y python-pip redis-server &&
                git clone --quiet ${repository} &&
                cd bugbug &&
                git -c advice.detachedHead=false checkout ${head_rev} &&
                pip install --disable-pip-version-check --quiet --no-cache-dir . &&
                python -c 'import os; print(os.environ.keys())' &&
                bash ./scripts/integration_test.sh"
            cache:
              bugbug-mercurial-repository: /cache
          metadata:
            name: bugbug integration test
            description: bugbug integration test
            owner: ${user}@users.noreply.github.com
            source: ${repository}/raw/${head_rev}/.taskcluster.yml