Bug 1386149 - Allow build jobs to use toolchain aliases instead of toolchain job names. r=dustin

It is desirable for e.g. smooth toolchain transitions, to be able to
refer them with generic name from toolchain jobs, while they have more
specific names, including version numbers. For example, in a near
future, there could be a linux64-gcc-4.9 toolchain and a linux64-gcc-6.
The default would be former, but at some point we'd want to switch to
the latter, without having to change all the toolchain definitions.

Moreover, when the switch happens, it would be desirable to have some
jobs stick with the old version, which is hard to keep track of when
all the toolchain definitions for build jobs use the same versioned
toolchain. With an alias, jobs that want the default use the alias, and
jobs that want to use a specific version use the versioned toolchain
name.

--HG--
extra : rebase_source : 467d713edd00dbe358483f5ee749fa56900714dd

taskcluster/docs/attributes.rst

@@ -169,3 +169,8 @@ This is the type of repackage. Can be ``repackage`` or
 toolchain-artifact
 ==================
 For toolchain jobs, this is the path to the artifact for that toolchain.
+
+toolchain-alias
+===============
+For toolchain jobs, this optionally gives an alias that can be used instead of the
+real toolchain job name in the toolchains list for build jobs.

taskcluster/taskgraph/transforms/job/toolchain.py

@@ -43,6 +43,10 @@ toolchain_run_schema = Schema({
 
     # Path to the artifact produced by the toolchain job
     Required('toolchain-artifact'): basestring,
+
+    # An alias that can be used instead of the real toolchain job name in
+    # the toolchains list for build jobs.
+    Optional('toolchain-alias'): basestring,
 })
 
 
@@ -132,6 +136,8 @@ def docker_worker_toolchain(config, job, taskdesc):
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
+    if 'toolchain-alias' in run:
+        attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimizations(config, run, taskdesc)
 
@@ -187,5 +193,7 @@ def windows_toolchain(config, job, taskdesc):
 
     attributes = taskdesc.setdefault('attributes', {})
     attributes['toolchain-artifact'] = run['toolchain-artifact']
+    if 'toolchain-alias' in run:
+        attributes['toolchain-alias'] = run['toolchain-alias']
 
     add_optimizations(config, run, taskdesc)

taskcluster/taskgraph/transforms/toolchain.py

@@ -16,27 +16,45 @@ def use_toolchains(config, jobs):
     of corresponding artifacts to jobs using toolchains.
     """
     artifacts = {}
+    aliases_by_job = {}
+
+    def get_attribute(dict, key, attributes, attribute_name):
+        '''Get `attribute_name` from the given `attributes` dict, and if there
+        is a corresponding value, set `key` in `dict` to that value.'''
+        value = attributes.get(attribute_name)
+        if value:
+            dict[key] = value
+
     # Toolchain jobs can depend on other toolchain jobs, but we don't have full
     # tasks for them, since they're being transformed. So scan the jobs list in
     # that case, otherwise, use the list of tasks for the kind dependencies.
     if config.kind == 'toolchain':
         jobs = list(jobs)
         for job in jobs:
-            artifact = job.get('run', {}).get('toolchain-artifact')
-            if artifact:
-                artifacts[job['name']] = artifact
+            run = job.get('run', {})
+            get_attribute(artifacts, job['name'], run, 'toolchain-artifact')
+            get_attribute(aliases_by_job, job['name'], run, 'toolchain-alias')
     else:
         for task in config.kind_dependencies_tasks:
             if task.kind != 'toolchain':
                 continue
-            artifact = task.attributes.get('toolchain-artifact')
-            if artifact:
-                artifacts[task.label.replace('%s-' % task.kind, '')] = artifact
+            name = task.label.replace('%s-' % task.kind, '')
+            get_attribute(artifacts, name, task.attributes, 'toolchain-artifact')
+            get_attribute(aliases_by_job, name, task.attributes, 'toolchain-alias')
+
+    aliases = {}
+    for job, alias in aliases_by_job.items():
+        if alias in aliases:
+            raise Exception(
+                "Cannot use the alias %s for %s, it's already used for %s"
+                % (alias, job, aliases[alias]))
+        aliases[alias] = job
 
     for job in jobs:
         env = job.setdefault('worker', {}).setdefault('env', {})
 
-        toolchains = job.pop('toolchains', [])
+        toolchains = [aliases.get(t, t)
+                      for t in job.pop('toolchains', [])]
 
         if config.kind == 'toolchain' and job['name'] in toolchains:
             raise Exception("Toolchain job %s can't use itself as toolchain"