Bug 1492664 - add {artifact-reference: ..} r=tomprince

This provides an easy way to encode an artifact URL in static data such as
taskcluster/ci/nightly-l10n/kind.yml.  This could be used in
mozharness_test.py, for example, as well -- but other code (such as to support
backfilling) expects `task-reference` there.  To avoid breaking such subtle
bits, those can continue using `task-reference` with URLs generated based on
TASKCLUSTER_ROOT_URL.

Differential Revision: https://phabricator.services.mozilla.com/D14197

--HG--
extra : moz-landing-system : lando

taskcluster/docs/optimization-process.rst

@@ -70,6 +70,6 @@ label. At this phase, the edges in the task graph diverge from the
 ``task.dependencies`` attributes, as the latter may contain dependencies
 outside of the taskgraph (for replacement tasks).
 
-As a side-effect, this phase also expands all ``{"task-reference": ".."}``
-objects within the task definitions.
+As a side-effect, this phase also expands all ``{"task-reference": ".."}`` and
+``{"artifact-reference": ".."}`` objects within the task definitions.
 

taskcluster/docs/taskgraph.rst

@@ -176,6 +176,11 @@ using simple parameterized values, as follows:
     Multiple labels may be substituted in a single string, and ``<<>`` can be
     used to escape a literal ``<``.
 
+``{"artifact-reference": "..<dep-name/artifact/name>.."}``
+    Similar to a ``task-reference``, but this substitutes a URL to the queue's
+    ``getLatestArtifact`` API method (for which a GET will redirect to the
+    artifact itself).
+
 .. _taskgraph-graph-config:
 
 Graph Configuration

taskcluster/docs/transforms.rst

@@ -202,9 +202,10 @@ this common functionality.  They expect a "signing description", and produce a
 task definition.  The schema for a signing description is defined at the top of
 ``signing.py``, with copious comments.
 
-In particular you define a set of upstream artifact urls (that point at the dependent
-task) and can optionally provide a dependent name (defaults to build) for use in
-task-reference. You also need to provide the signing formats to use.
+In particular you define a set of upstream artifact urls (that point at the
+dependent task) and can optionally provide a dependent name (defaults to build)
+for use in ``task-reference``/``artifact-reference``. You also need to provide
+the signing formats to use.
 
 More Detail
 -----------

taskcluster/taskgraph/morph.py

@@ -11,10 +11,11 @@ locally, so they should be limited to changes that do not modify the meaning of
 the graph.
 """
 
-# Note that the translation of `{'task-reference': '..'}` is handled in the
-# optimization phase (since optimization involves dealing with taskIds
-# directly).  Similarly, `{'relative-datestamp': '..'}` is handled at the last
-# possible moment during task creation.
+# Note that the translation of `{'task-reference': '..'}` and
+# `artifact-reference` are handled in the optimization phase (since
+# optimization involves dealing with taskIds directly).  Similarly,
+# `{'relative-datestamp': '..'}` is handled at the last possible moment during
+# task creation.
 
 from __future__ import absolute_import, print_function, unicode_literals
 

taskcluster/taskgraph/test/test_util_parameterization.py

@@ -6,6 +6,8 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 import datetime
+import mock
+import os
 
 from mozunit import main
 from taskgraph.util.parameterization import (
@@ -88,5 +90,43 @@ class TestTaskRefs(unittest.TestCase):
         )
 
 
+class TestArtifactRefs(unittest.TestCase):
+
+    def do(self, input, output):
+        taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
+        with mock.patch.dict(os.environ, {'TASKCLUSTER_ROOT_URL': 'https://tc-tests.localhost'}):
+            self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name),
+                             output)
+
+    def test_in_list(self):
+        "resolve_task_references resolves artifact references in a list"
+        self.do(
+            {'in-a-list': [
+                'stuff', {'artifact-reference': '<edge1/foo/bar>'}]},
+            {'in-a-list': [
+                'stuff', 'https://tc-tests.localhost/api/queue/v1/task/tid1/artifacts/foo/bar']})
+
+    def test_in_dict(self):
+        "resolve_task_references resolves artifact references in a dict"
+        self.do(
+            {'in-a-dict':
+                {'stuff': {'artifact-reference': '<edge2/bar/foo>'}}},
+            {'in-a-dict':
+                {'stuff': 'https://tc-tests.localhost/api/queue/v1/task/tid2/artifacts/bar/foo'}})
+
+    def test_in_string(self):
+        "resolve_task_references resolves artifact references embedded in a string"
+        self.do(
+            {'stuff': {'artifact-reference': '<edge1/filename> and <edge2/bar/foo>'}},
+            {'stuff': 'https://tc-tests.localhost/api/queue/v1/task/tid1/artifacts/filename and '
+                'https://tc-tests.localhost/api/queue/v1/task/tid2/artifacts/bar/foo'})
+
+    def test_invalid(self):
+        "resolve_task_references ignores badly-formatted artifact references"
+        for inv in ['<edge1>', 'edge1/foo>', '<edge1>/foo', '<edge1>foo']:
+            resolved = resolve_task_references('subject', {'artifact-reference': inv}, {})
+            self.assertEqual(resolved, inv)
+
+
 if __name__ == '__main__':
     main()

taskcluster/taskgraph/transforms/balrog_submit.py

@@ -17,18 +17,13 @@ from taskgraph.util.scriptworker import (
     get_balrog_server_scope, get_worker_type_for_scope
 )
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 balrog_description_schema = schema.extend({
     # unique label to describe this balrog task, defaults to balrog-{dep.label}
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/beetmover.py

@@ -7,7 +7,7 @@ Transform the beetmover task into an actual task description.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from voluptuous import Any, Optional, Required
+from voluptuous import Optional, Required
 
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
@@ -119,11 +119,6 @@ task_description_schema = {str(k): v for k, v in task_description_schema.schema.
 
 transforms = TransformSequence()
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,

taskcluster/taskgraph/transforms/beetmover_checksums.py

@@ -18,17 +18,13 @@ from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
                                          get_beetmover_bucket_scope,
                                          get_worker_type_for_scope,
                                          should_use_artifact_map)
-from voluptuous import Any, Optional, Required
+from voluptuous import Optional, Required
 from taskgraph.transforms.task import task_description_schema
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_checksums_description_schema = schema.extend({
     Required('depname', default='build'): basestring,
     Required('attributes'): {basestring: object},

taskcluster/taskgraph/transforms/beetmover_emefree_checksums.py

@@ -12,16 +12,12 @@ from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.beetmover import craft_release_properties
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_checksums_description_schema = schema.extend({
     Required('depname', default='build'): basestring,
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py

@@ -15,16 +15,12 @@ from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope,
                                          get_worker_type_for_scope)
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_checksums_description_schema = schema.extend({
     Required('depname', default='build'): basestring,
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/beetmover_push_to_release.py

@@ -8,14 +8,17 @@ Transform the beetmover-push-to-release task into a task description.
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import Schema
+from taskgraph.util.schema import (
+    Schema,
+    taskref_or_string,
+)
 from taskgraph.util.scriptworker import (
     get_beetmover_bucket_scope, add_scope_prefix,
     get_worker_type_for_scope,
 )
 from taskgraph.transforms.job import job_description_schema
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -23,10 +26,6 @@ task_description_schema = {str(k): v for k, v in task_description_schema.schema.
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
 
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_push_to_release_description_schema = Schema({
     Required('name'): basestring,
     Required('product'): basestring,

taskcluster/taskgraph/transforms/beetmover_repackage.py

@@ -19,7 +19,7 @@ from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_worker_type_for_scope)
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 import logging
 import re
@@ -146,11 +146,6 @@ UPSTREAM_ARTIFACT_SIGNED_MAR_PATHS = [
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,

taskcluster/taskgraph/transforms/beetmover_repackage_partner.py

@@ -39,11 +39,6 @@ logger = logging.getLogger(__name__)
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the unsigned things
     Required('depname', default='build'): basestring,

taskcluster/taskgraph/transforms/beetmover_source_checksums.py

@@ -18,16 +18,12 @@ from taskgraph.util.scriptworker import (generate_beetmover_artifact_map,
                                          get_worker_type_for_scope,
                                          should_use_artifact_map)
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 beetmover_checksums_description_schema = schema.extend({
     Required('depname', default='build'): basestring,
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/checksums_signing.py

@@ -16,16 +16,12 @@ from taskgraph.util.scriptworker import (
     add_scope_prefix,
 )
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 checksums_signing_description_schema = schema.extend({
     Required('depname', default='beetmover'): basestring,
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/l10n.py

@@ -18,6 +18,7 @@ from taskgraph.transforms.base import (
 from taskgraph.util.schema import (
     optionally_keyed_by,
     resolve_keyed_by,
+    taskref_or_string,
 )
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.taskcluster import get_artifact_prefix
@@ -35,11 +36,6 @@ def _by_platform(arg):
     return optionally_keyed_by('build-platform', arg)
 
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}

taskcluster/taskgraph/transforms/repackage.py

@@ -20,18 +20,13 @@ from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.platforms import archive_format, executable_extension
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.job import job_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
 
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 packaging_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the signed things
     Required('depname', default='build'): basestring,

taskcluster/taskgraph/transforms/repackage_partner.py

@@ -22,7 +22,7 @@ from taskgraph.util.platforms import archive_format, executable_extension
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from taskgraph.transforms.repackage import PACKAGE_FORMATS
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
@@ -33,11 +33,6 @@ def _by_platform(arg):
     return optionally_keyed_by('build-platform', arg)
 
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 packaging_description_schema = schema.extend({
     # depname is used in taskref's to identify the taskID of the signed things
     Required('depname', default='build'): basestring,

taskcluster/taskgraph/transforms/signing.py

@@ -10,13 +10,14 @@ from __future__ import absolute_import, print_function, unicode_literals
 from taskgraph.loader.single_dep import schema
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.schema import taskref_or_string
 from taskgraph.util.scriptworker import (
     add_scope_prefix,
     get_signing_cert_scope_per_platform,
     get_worker_type_for_scope,
 )
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
@@ -25,11 +26,6 @@ task_description_schema = {str(k): v for k, v in task_description_schema.schema.
 
 transforms = TransformSequence()
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 signing_description_schema = schema.extend({
     # Artifacts from dep task to sign - Sync with taskgraph/transforms/task.py
     # because this is passed directly into the signingscript worker

taskcluster/taskgraph/transforms/source_checksums_signing.py

@@ -16,16 +16,12 @@ from taskgraph.util.scriptworker import (
     add_scope_prefix,
 )
 from taskgraph.transforms.task import task_description_schema
-from voluptuous import Any, Required, Optional
+from voluptuous import Required, Optional
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring})
-
 checksums_signing_description_schema = schema.extend({
     Required('depname', default='beetmover'): basestring,
     Optional('label'): basestring,

taskcluster/taskgraph/transforms/task.py

@@ -27,6 +27,7 @@ from taskgraph.util.schema import (
     optionally_keyed_by,
     resolve_keyed_by,
     OptimizationSchema,
+    taskref_or_string,
 )
 from taskgraph.util.scriptworker import (
     BALROG_ACTIONS,
@@ -47,12 +48,6 @@ def _run_task_suffix():
     return hash_path(RUN_TASK)[0:20]
 
 
-# shortcut for a string where task references are allowed
-taskref_or_string = Any(
-    basestring,
-    {Required('task-reference'): basestring},
-)
-
 # A task description is a general description of a TaskCluster task
 task_description_schema = Schema({
     # the label for this task

taskcluster/taskgraph/util/parameterization.py

@@ -4,24 +4,26 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import os
 import re
+import taskcluster_urls
 
 from taskgraph.util.time import json_time_from_now
 
 TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
+ARTIFACT_REFERENCE_PATTERN = re.compile('<([^/]+)/([^>]+)>')
 
 
-def _recurse(val, param_name, param_fn):
-    param_keys = [param_name]
-
+def _recurse(val, param_fns):
     def recurse(val):
         if isinstance(val, list):
             return [recurse(v) for v in val]
         elif isinstance(val, dict):
-            if val.keys() == param_keys:
-                return param_fn(val[param_name])
-            else:
-                return {k: recurse(v) for k, v in val.iteritems()}
+            if len(val) == 1:
+                for param_key, param_fn in param_fns.items():
+                    if val.keys() == [param_key]:
+                        return param_fn(val[param_key])
+            return {k: recurse(v) for k, v in val.iteritems()}
         else:
             return val
     return recurse(val)
@@ -29,20 +31,47 @@ def _recurse(val, param_name, param_fn):
 
 def resolve_timestamps(now, task_def):
     """Resolve all instances of `{'relative-datestamp': '..'}` in the given task definition"""
-    return _recurse(task_def, 'relative-datestamp', lambda v: json_time_from_now(v, now))
+    return _recurse(task_def, {
+        'relative-datestamp': lambda v: json_time_from_now(v, now),
+    })
 
 
 def resolve_task_references(label, task_def, dependencies):
-    """Resolve all instances of `{'task-reference': '..<..>..'}` in the given task
-    definition, using the given dependencies"""
-    def repl(match):
-        key = match.group(1)
-        try:
-            return dependencies[key]
-        except KeyError:
-            # handle escaping '<'
-            if key == '<':
-                return key
-            raise KeyError("task '{}' has no dependency named '{}'".format(label, key))
+    """Resolve all instances of
+      {'task-reference': '..<..>..'}
+    and
+      {'artifact-reference`: '..<dependency/artifact/path>..'}
+    in the given task definition, using the given dependencies"""
 
-    return _recurse(task_def, 'task-reference', lambda v: TASK_REFERENCE_PATTERN.sub(repl, v))
+    def task_reference(val):
+        def repl(match):
+            key = match.group(1)
+            try:
+                return dependencies[key]
+            except KeyError:
+                # handle escaping '<'
+                if key == '<':
+                    return key
+                raise KeyError("task '{}' has no dependency named '{}'".format(label, key))
+
+        return TASK_REFERENCE_PATTERN.sub(repl, val)
+
+    def artifact_reference(val):
+        def repl(match):
+            dependency, artifact_name = match.group(1, 2)
+
+            try:
+                dependency_task_id = dependencies[dependency]
+            except KeyError:
+                raise KeyError("task '{}' has no dependency named '{}'".format(label, dependency))
+
+            return taskcluster_urls.api(
+                os.environ['TASKCLUSTER_ROOT_URL'], 'queue', 'v1',
+                'task/{}/artifacts/{}'.format(dependency_task_id, artifact_name))
+
+        return ARTIFACT_REFERENCE_PATTERN.sub(repl, val)
+
+    return _recurse(task_def, {
+        'task-reference': task_reference,
+        'artifact-reference': artifact_reference,
+    })

taskcluster/taskgraph/util/schema.py

@@ -234,3 +234,10 @@ OptimizationSchema = voluptuous.Any(
     # are unnecessary if the parent tasks are not run)
     {'only-if-dependencies-run': None}
 )
+
+# shortcut for a string where task references are allowed
+taskref_or_string = voluptuous.Any(
+    basestring,
+    {voluptuous.Required('task-reference'): basestring},
+    {voluptuous.Required('artifact-reference'): basestring},
+)