Bug 1302590 - remove nightly-fennec kind. use build kind for nightlies, r=dustin

* add fennec nightly build to job whitelist MozReview-Commit-ID: F8MCUWwlFYD --HG-- rename : taskcluster/ci/signing/signing.yml => taskcluster/ci/build-signing/android-signing.yml rename : taskcluster/ci/signing/kind.yml => taskcluster/ci/build-signing/kind.yml extra : rebase_source : 5c6095daaa6d8994d7bdf0d2cc2770e41aca73ca extra : source : 1b269fb8a704a1a1fd2fe67543b9a083ce155563
2016-09-16 01:20:38 -07:00 · 2016-09-16 01:20:38 -07:00 · eec7a1c8df
--- a/taskcluster/ci/build-signing/android-signing.yml
+++ b/taskcluster/ci/build-signing/android-signing.yml
@ -0,0 +1,26 @@
+signing-nightly-fennec:
+  task:
+    provisionerId: "scriptworker-prov-v1"
+    workerType: "signing-linux-v1"
+    scopes:
+      - "project:releng:signing:cert:dep-signing"
+      - "project:releng:signing:format:jar"
+    created:
+      relative-datestamp: "0 seconds"
+    deadline:
+      relative-datestamp: "24 hours"
+    payload:
+      unsignedArtifacts: []
+      maxRunTime: 600
+    metadata:
+      name: "Signing Scriptworker Task"
+      description: "Sign Android Build Tasks"
+      owner: "jlund@mozilla.com"
+      source: "https://tools.taskcluster.net/task-creator/"
+  attributes:
+    nightly: true
+  unsigned-task:
+    label: "build-android-api-15-nightly/opt"
+    artifacts:
+      - "public/build/target.apk"
+      - "public/build/en-US/target.apk"
--- a/taskcluster/ci/build-signing/kind.yml
+++ b/taskcluster/ci/build-signing/kind.yml
@ -3,6 +3,9 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

 implementation: 'taskgraph.task.signing:SigningTask'
-signing_path: '.'
+
+jobs-from:
+  - android-signing.yml
+
 kind-dependencies:
-  - nightly-fennec
+  - build
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@ -72,6 +72,34 @@ android-api-15/opt:
        custom-build-variant-cfg: api-15
        tooltool-downloads: internal

+android-api-15-nightly/opt:
+    description: "Android 4.0 API15+ Nightly"
+    attributes:
+        nightly: true
+    index:
+        product: mobile
+        job-name: android-api-15-nightly-opt
+    treeherder:
+        platform: android-4-0-armv7-api15/opt
+        symbol: tc(N)
+        tier: 2
+    worker-type: aws-provisioner-v1/android-api-15
+    worker:
+        implementation: docker-worker
+        max-run-time: 7200
+    run:
+        using: mozharness
+        actions: [get-secrets build multi-l10n update]
+        config:
+            - builds/releng_base_android_64_builds.py
+            - disable_signing.py
+            - platform_supports_post_upload_to_latest.py
+            - taskcluster_nightly.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: api-15
+        tooltool-downloads: internal
+    run-on-projects: []
+
 android-api-15-gradle/opt:
    description: "Android 4.0 API15+ (Gradle) Opt"
    index:
--- a/taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
+++ b/taskcluster/ci/nightly-fennec/android-api-15-nightly-build.yml
@ -1,66 +0,0 @@
-$inherits:
-  from: 'mobile_base.yml'
-  variables:
-    build_name: 'android'
-    build_type: 'opt'
-task:
-  metadata:
-      name: '[TC] Android armv7 API 15+'
-      description: 'Android armv7 API 15+'
-
-  workerType: android-api-15
-
-  routes:
-    - 'index.buildbot.branches.{{project}}.android-api-15'
-    - 'index.buildbot.revisions.{{head_rev}}.{{project}}.android-api-15'
-
-  scopes:
-    - 'docker-worker:cache:level-{{level}}-{{project}}-build-android-api-15-workspace'
-    - 'docker-worker:cache:tooltool-cache'
-    - 'docker-worker:relengapi-proxy:tooltool.download.internal'
-    - 'docker-worker:relengapi-proxy:tooltool.download.public'
-
-  payload:
-    cache:
-      level-{{level}}-{{project}}-build-android-api-15-workspace: '/home/worker/workspace'
-      tooltool-cache: '/home/worker/tooltool-cache'
-
-    features:
-      relengAPIProxy: true
-
-    env:
-      # inputs to mozharness
-      MOZHARNESS_SCRIPT: 'mozharness/scripts/fx_desktop_build.py'
-      # TODO: make these additional configuration files go away
-      MOZHARNESS_CONFIG: >
-          builds/releng_base_android_64_builds.py
-          disable_signing.py
-          platform_supports_post_upload_to_latest.py
-          taskcluster_nightly.py
-      MOZHARNESS_ACTIONS: "get-secrets build multi-l10n update"
-      MH_CUSTOM_BUILD_VARIANT_CFG: api-15
-      MH_BRANCH: {{project}}
-      MH_BUILD_POOL: taskcluster
-      TOOLTOOL_CACHE: '/home/worker/tooltool-cache'
-
-    command: ["/bin/bash", "bin/build.sh"]
-
-  extra:
-    treeherderEnv:
-      - production
-      - staging
-    treeherder:
-      machine:
-        # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
-        platform: android-4-0-armv7-api15
-      groupSymbol: tc
-      groupName: Submitted by taskcluster
-      symbol: B
-      tier: 2
-    # Rather then enforcing particular conventions we require that all build
-    # tasks provide the "build" extra field to specify where the build and tests
-    # files are located.
-    locations:
-      build: 'public/build/target.apk'
-      mozharness: 'public/build/mozharness.zip'
-      test_packages: 'public/build/target.test_packages.json'
--- a/taskcluster/ci/nightly-fennec/build.yml
+++ b/taskcluster/ci/nightly-fennec/build.yml
@ -1,46 +0,0 @@
-# This is the "base" task which contains the common values all builds must
-# provide.
---
-taskId: {{build_slugid}}
-
-task:
-  created:
-    relative-datestamp: "0 seconds"
-  deadline:
-    relative-datestamp: "24 hours"
-  metadata:
-    source: '{{source}}'
-    owner: mozilla-taskcluster-maintenance@mozilla.com
-
-  tags:
-    createdForUser: {{owner}}
-
-  provisionerId: aws-provisioner-v1
-  schedulerId: task-graph-scheduler
-
-  payload:
-    # Two hours is long but covers edge cases (and matches bb based infra)
-    maxRunTime: 7200
-
-    env:
-      # Common environment variables for checking out gecko
-      GECKO_BASE_REPOSITORY: '{{base_repository}}'
-      GECKO_HEAD_REPOSITORY: '{{head_repository}}'
-      GECKO_HEAD_REV: '{{head_rev}}'
-      GECKO_HEAD_REF: '{{head_ref}}'
-      TOOLTOOL_REPO: 'https://github.com/mozilla/build-tooltool'
-      TOOLTOOL_REV: 'master'
-      MOZ_BUILD_DATE: '{{pushdate}}'
-      MOZ_SCM_LEVEL: '{{level}}'
-
-  extra:
-    build_product: '{{build_product}}'
-    build_name: '{{build_name}}'
-    build_type: '{{build_type}}'
-    index:
-      rank: {{rank}}
-    treeherder:
-      jobKind: build
-      groupSymbol: tc
-      groupName: Submitted by taskcluster
-      symbol: B
--- a/taskcluster/ci/nightly-fennec/docker_build.yml
+++ b/taskcluster/ci/nightly-fennec/docker_build.yml
@ -1,27 +0,0 @@
-$inherits:
-  from: 'build.yml'
-
-
-task:
-  workerType: b2gbuild
-
-  routes:
-    - 'index.gecko.v1.{{project}}.revision.linux.{{head_rev}}.{{build_name}}.{{build_type}}'
-    - 'index.gecko.v1.{{project}}.latest.linux.{{build_name}}.{{build_type}}'
-
-  scopes:
-    # docker build tasks use tc-vcs so include the scope.
-    - 'docker-worker:cache:level-{{level}}-{{project}}-tc-vcs'
-
-  payload:
-
-    cache:
-      level-{{level}}-{{project}}-tc-vcs: '/home/worker/.tc-vcs'
-
-    # All docker builds share a common artifact directory for ease of uploading.
-    artifacts:
-      'public/build':
-        type: directory
-        path: '/home/worker/artifacts/'
-        expires:
-          relative-datestamp: '1 year'
--- a/taskcluster/ci/nightly-fennec/kind.yml
+++ b/taskcluster/ci/nightly-fennec/kind.yml
@ -1,7 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-implementation: 'taskgraph.task.nightly_fennec:NightlyFennecTask'
-nightly_fennec_path: '.'
-
--- a/taskcluster/ci/nightly-fennec/mobile_base.yml
+++ b/taskcluster/ci/nightly-fennec/mobile_base.yml
@ -1,13 +0,0 @@
-$inherits:
-  from: 'docker_build.yml'
-  variables:
-    build_product: 'mobile'
-docker-image: desktop-build
-task:
-  payload:
-    image:
-      type: 'task-image'
-      path: 'public/image.tar'
-      taskId:
-        task-reference: "<docker-image>"
-
--- a/taskcluster/ci/nightly-fennec/routes.json
+++ b/taskcluster/ci/nightly-fennec/routes.json
@ -1,18 +0,0 @@
-{
-    "routes": [
-        "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
-        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
-        "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
-    ],
-    "nightly": [
-        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
-        "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
-        "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
-        "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
-    ],
-    "l10n": [
-        "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
-        "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
-        "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
-    ]
-}
--- a/taskcluster/ci/signing/signing.yml
+++ b/taskcluster/ci/signing/signing.yml
@ -1,18 +0,0 @@
-task:
-  provisionerId: "scriptworker-prov-v1"
-  workerType: "signing-linux-v1"
-  scopes:
-    - "project:releng:signing:cert:dep-signing"
-    - "project:releng:signing:format:jar"
-  created:
-    relative-datestamp: "0 seconds"
-  deadline:
-    relative-datestamp: "24 hours"
-  payload:
-    unsignedArtifacts: []
-    maxRunTime: 600
-  metadata:
-    name: "Signing Scriptworker Task"
-    description: "Testing the signing scriptworker"
-    owner: "amiyaguchi@mozilla.com"
-    source: "https://tools.taskcluster.net/task-creator/"
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@ -116,3 +116,9 @@ image_name
 ==========

 For the ``docker_image`` kind, this attribute contains the docker image name.
+
+nightly
+=======
+
+Signals whether the task is part of a nightly graph. Useful when filtering
+out nightly tasks from full task set at target stage.
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@ -106,5 +106,6 @@ def target_tasks_nightly(full_task_graph, parameters):
    """Select the set of tasks required for a nightly build of fennec. The
    nightly build process involves a pipeline of builds, signing,
    and, eventually, uploading the tasks to balrog."""
-    return [t.label for t in full_task_graph.tasks.itervalues()
-            if t.attributes.get('kind') in ['nightly-fennec', 'signing']]
+    def filter(task):
+        return task.attributes.get('nightly', False)
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
--- a/taskcluster/taskgraph/task/nightly_fennec.py
+++ b/taskcluster/taskgraph/task/nightly_fennec.py
@ -1,201 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-Temporary placeholder for nightly builds
-"""
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import logging
-import json
-import os
-import time
-from collections import namedtuple
-
-from . import base
-from slugid import nice as slugid
-from taskgraph.util.templates import Templates
-from taskgraph.util.docker import docker_image
-
-
-logger = logging.getLogger(__name__)
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
-INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
-TASKID_PLACEHOLDER = 'TaskLabel=={}'
-
-
-def mklabel():
-    return TASKID_PLACEHOLDER.format(slugid())
-
-
-def gaia_info():
-    '''Fetch details from in tree gaia.json (which links this version of
-    gecko->gaia) and construct the usual base/head/ref/rev pairing...'''
-    gaia = json.load(open(os.path.join(GECKO, 'b2g', 'config', 'gaia.json')))
-
-    if gaia['git'] is None or \
-       gaia['git']['remote'] == '' or \
-       gaia['git']['git_revision'] == '' or \
-       gaia['git']['branch'] == '':
-
-        # Just use the hg params...
-        return {
-            'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-            'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
-            'gaia_ref': gaia['revision'],
-            'gaia_rev': gaia['revision']
-        }
-
-    else:
-        # Use git
-        return {
-            'gaia_base_repository': gaia['git']['remote'],
-            'gaia_head_repository': gaia['git']['remote'],
-            'gaia_rev': gaia['git']['git_revision'],
-            'gaia_ref': gaia['git']['branch'],
-        }
-
-
-def decorate_task_json_routes(task, json_routes, parameters):
-    """Decorate the given task with routes.json routes.
-
-    :param dict task: task definition.
-    :param json_routes: the list of routes to use from routes.json
-    :param parameters: dictionary of parameters to use in route templates
-    """
-    routes = task.get('routes', [])
-    for route in json_routes:
-        routes.append(route.format(**parameters))
-
-    task['routes'] = routes
-
-
-def query_vcs_info(repository, revision):
-    """Query the pushdate and pushid of a repository/revision.
-
-    This is intended to be used on hg.mozilla.org/mozilla-central and
-    similar. It may or may not work for other hg repositories.
-    """
-    if not repository or not revision:
-        logger.warning('cannot query vcs info because vcs info not provided')
-        return None
-
-    VCSInfo = namedtuple('VCSInfo', ['pushid', 'pushdate', 'changesets'])
-
-    try:
-        import requests
-        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'),
-                                                  revision)
-        logger.debug("Querying version control for metadata: %s", url)
-        contents = requests.get(url).json()
-
-        changesets = []
-        for c in contents['changesets']:
-            changesets.append({k: c[k] for k in ('desc', 'files', 'node')})
-
-        pushid = contents['changesets'][-1]['pushid']
-        pushdate = contents['changesets'][-1]['pushdate'][0]
-
-        return VCSInfo(pushid, pushdate, changesets)
-
-    except Exception:
-        logger.exception("Error querying VCS info for '%s' revision '%s'",
-                         repository, revision)
-        return None
-
-
-class NightlyFennecTask(base.Task):
-
-    def __init__(self, *args, **kwargs):
-        try:
-            self.task_dict = kwargs.pop('task_dict')
-        except KeyError:
-            pass
-        super(NightlyFennecTask, self).__init__(*args, **kwargs)
-
-    @classmethod
-    def load_tasks(cls, kind, path, config, params, loaded_tasks):
-        root = os.path.abspath(os.path.join(path, config[
-            'nightly_fennec_path']))
-
-        project = params['project']
-
-        # Set up the parameters, including the time of the build
-        push_epoch = int(time.time())
-        vcs_info = query_vcs_info(params['head_repository'], params['head_rev'])
-        changed_files = set()
-        if vcs_info:
-            push_epoch = vcs_info.pushdate
-
-            logger.debug(
-                '{} commits influencing task scheduling:'.format(len(vcs_info.changesets)))
-            for c in vcs_info.changesets:
-                logger.debug("{cset} {desc}".format(
-                    cset=c['node'][0:12],
-                    desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
-                changed_files |= set(c['files'])
-
-        pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(push_epoch))
-
-        # Template parameters used when expanding the graph
-        parameters = dict(gaia_info().items() + {
-            'index': 'index',
-            'project': project,
-            'pushlog_id': params.get('pushlog_id', 0),
-            'base_repository': params['base_repository'] or params['head_repository'],
-            'docker_image': docker_image,
-            'head_repository': params['head_repository'],
-            'head_ref': params['head_ref'] or params['head_rev'],
-            'head_rev': params['head_rev'],
-            'pushdate': pushdate,
-            'pushtime': pushdate[8:],
-            'year': pushdate[0:4],
-            'month': pushdate[4:6],
-            'day': pushdate[6:8],
-            'rank': push_epoch,
-            'owner': params['owner'],
-            'level': params['level'],
-            'build_slugid': mklabel(),
-            'source': '{repo}file/{rev}/taskcluster/ci/nightly-fennec'.format(
-                repo=params['head_repository'], rev=params['head_rev']),
-            'build_name': 'android',
-            'build_type': 'opt',
-            'build_product': 'mobile'
-        }.items())
-
-        routes_file = os.path.join(root, 'routes.json')
-        with open(routes_file) as f:
-            contents = json.load(f)
-            json_routes = contents['routes']
-
-        tasks = []
-        templates = Templates(root)
-
-        task = templates.load('android-api-15-nightly-build.yml', parameters)
-        decorate_task_json_routes(task['task'], json_routes, parameters)
-
-        attributes = {'kind': 'nightly-fennec'}
-        tasks.append(cls(kind, 'build-nightly-fennec',
-                     task=task['task'], attributes=attributes,
-                         task_dict=task))
-
-        # Convert to a dictionary of tasks.  The process above has invented a
-        # taskId for each task, and we use those as the *labels* for the tasks;
-        # taskgraph will later assign them new taskIds.
-        return tasks
-
-    def get_dependencies(self, taskgraph):
-        deps = [(label, label) for label in self.task_dict.get('requires', [])]
-
-        # add a dependency on an image task, if needed
-        if 'docker-image' in self.task_dict:
-            deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
-                         'docker-image'))
-
-        return deps
-
-    def optimize(self, params):
-        return False, None
--- a/taskcluster/taskgraph/task/signing.py
+++ b/taskcluster/taskgraph/task/signing.py
@ -19,37 +19,34 @@ INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'

 class SigningTask(base.Task):

-    def __init__(self, *args, **kwargs):
-        super(SigningTask, self).__init__(*args, **kwargs)
+    def __init__(self, kind, name, task, attributes):
+        self.unsigned_artifact_label = task['unsigned-task']['label']
+        super(SigningTask, self).__init__(kind, name, task=task['task'],
+                                          attributes=attributes)

    @classmethod
    def load_tasks(cls, kind, path, config, params, loaded_tasks):
-        root = os.path.abspath(os.path.join(path, config['signing_path']))
-
-        # get each nightly-fennec and add its name to this task
-        fennec_tasks = [t for t in loaded_tasks if t.attributes.get('kind') == 'nightly-fennec']
+        root = os.path.abspath(path)

        tasks = []
-        for fennec_task in fennec_tasks:
+        for filename in config.get('jobs-from', []):
            templates = Templates(root)
-            task = templates.load('signing.yml', {})
+            jobs = templates.load(filename, {})

-            artifacts = ['public/build/target.apk',
-                         'public/build/en-US/target.apk']
-            for artifact in artifacts:
-                url = ARTIFACT_URL.format('<build-nightly-fennec>', artifact)
-                task['task']['payload']['unsignedArtifacts'].append({
-                    'task-reference': url
-                })
-
-            attributes = {'kind': 'signing'}
-            tasks.append(cls(kind, 'signing-nightly-fennec', task=task['task'],
-                             attributes=attributes))
+            for name, job in jobs.iteritems():
+                for artifact in job['unsigned-task']['artifacts']:
+                    url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
+                    job['task']['payload']['unsignedArtifacts'].append({
+                        'task-reference': url
+                    })
+                attributes = job.setdefault('attributes', {})
+                attributes.update({'kind': 'signing'})
+                tasks.append(cls(kind, name, job, attributes=attributes))

        return tasks

    def get_dependencies(self, taskgraph):
-        return [('build-nightly-fennec', 'build-nightly-fennec')]
+        return [(self.unsigned_artifact_label, 'unsigned-artifact')]

    def optimize(self, params):
        return False, None
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@ -20,6 +20,7 @@ JOB_NAME_WHITELIST = set([
    'android-api-15-gradle-dependencies-opt',
    'android-api-15-gradle-opt',
    'android-api-15-opt',
+    'android-api-15-nightly-opt',
    'android-api-15-partner-sample1-opt',
    'android-l10n-opt',
    'android-x86-opt',