Bug 1850045: replace command-context with task-context r=taskgraph-reviewers,releng-reviewers,gabriel

Most of this patch is ripping `command-context` out from Gecko. The other parts are the fairly straightforward conversions from `command-context` to `task-context`. Differential Revision: https://phabricator.services.mozilla.com/D186822
2023-09-07 15:42:09 +00:00 · 2023-09-07 15:42:09 +00:00 · 2a59fdc445
--- a/taskcluster/ci/attribution-l10n/kind.yml
+++ b/taskcluster/ci/attribution-l10n/kind.yml
@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader
 transforms:
    - gecko_taskgraph.transforms.split_by_locale:transforms
    - gecko_taskgraph.transforms.attribution:transforms
+    - taskgraph.transforms.task_context
    - gecko_taskgraph.transforms.job:transforms
    - gecko_taskgraph.transforms.task:transforms

@ -37,6 +38,10 @@ job-defaults:
    fetches:
        repackage-signing-l10n:
            - "{locale}/target.installer.exe"
+    task-context:
+        from-file: browser/installer/attribution.yml
+        substitution-fields:
+            - run.command
    worker:
        artifacts:
            - name: public/build
@ -62,8 +67,6 @@ job-defaults:
            # attribution of other files, which they can append themselves.
            - --input
            - /builds/worker/fetches/target.installer.exe
-        command-context:
-            from-file: browser/installer/attribution.yml

 jobs:
    win32-devedition/opt:
--- a/taskcluster/ci/attribution/kind.yml
+++ b/taskcluster/ci/attribution/kind.yml
@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader

 transforms:
    - gecko_taskgraph.transforms.attribution:transforms
+    - taskgraph.transforms.task_context
    - gecko_taskgraph.transforms.job:transforms
    - gecko_taskgraph.transforms.task:transforms

@ -29,6 +30,10 @@ job-defaults:
        symbol: Attr
        kind: other
        tier: 1
+    task-context:
+        from-file: browser/installer/attribution.yml
+        substitution-fields:
+            - run.command
    worker:
        artifacts:
            - name: public/build
@ -54,8 +59,6 @@ job-defaults:
            # attribution of other files, which they can append themselves.
            - --input
            - /builds/worker/fetches/target.installer.exe
-        command-context:
-            from-file: browser/installer/attribution.yml

 jobs:
    win32-devedition/opt:
--- a/taskcluster/ci/fuzzing/kind.yml
+++ b/taskcluster/ci/fuzzing/kind.yml
@ -6,6 +6,7 @@ loader: gecko_taskgraph.loader.transform:loader

 transforms:
    - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
    - gecko_taskgraph.transforms.job:transforms
    - gecko_taskgraph.transforms.task:transforms

@ -23,6 +24,9 @@ job-defaults:
    treeherder:
        kind: test
        tier: 3
+    task-context:
+        from-object: {}
+        substitution-fields: []
    if-dependencies: [build]

 jobs:
--- a/taskcluster/ci/perftest/kind.yml
+++ b/taskcluster/ci/perftest/kind.yml
@ -11,6 +11,7 @@ kind-dependencies:
 transforms:
    - gecko_taskgraph.transforms.perftest:transforms
    - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
    - gecko_taskgraph.transforms.job:transforms
    - gecko_taskgraph.transforms.task:transforms

@ -28,6 +29,9 @@ job-defaults:
    treeherder:
        kind: other
        tier: 3
+    task-context:
+        from-object: {}
+        substitution-fields: []
    worker:
        taskcluster-proxy: true
        max-run-time: 10800
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@ -7,6 +7,7 @@ loader: gecko_taskgraph.loader.transform:loader
 transforms:
    - gecko_taskgraph.transforms.try_job:transforms
    - gecko_taskgraph.transforms.source_test:transforms
+    - taskgraph.transforms.task_context
    - gecko_taskgraph.transforms.release_notifications:transforms
    - gecko_taskgraph.transforms.job:transforms
    - gecko_taskgraph.transforms.task:transforms
@ -38,3 +39,6 @@ jobs-from:
 job-defaults:
    attributes:
        retrigger: true
+    task-context:
+        from-object: {}
+        substitution-fields: []
--- a/taskcluster/gecko_taskgraph/test/test_transforms_job.py
+++ b/taskcluster/gecko_taskgraph/test/test_transforms_job.py
@ -107,44 +107,5 @@ def test_worker_caches(task, transform):
    validate_schema(partial_schema, taskdesc["worker"][key], "validation error")


-@pytest.mark.parametrize(
-    "workerfn", [fn for fn, *_ in job.registry["run-task"].values()]
-)
-@pytest.mark.parametrize(
-    "task",
-    (
-        {
-            "worker-type": "b-linux",
-            "run": {
-                "checkout": True,
-                "comm-checkout": False,
-                "command": "echo '{output}'",
-                "command-context": {"output": "hello", "extra": None},
-                "run-as-root": False,
-                "sparse-profile": False,
-                "tooltool-downloads": False,
-            },
-        },
-    ),
-)
-def test_run_task_command_context(task, transform, workerfn):
-    config, job_, taskdesc, _ = transform(task)
-    job_ = deepcopy(job_)
-
-    def assert_cmd(expected):
-        cmd = taskdesc["worker"]["command"]
-        while isinstance(cmd, list):
-            cmd = cmd[-1]
-        assert cmd == expected
-
-    workerfn(config, job_, taskdesc)
-    assert_cmd("echo 'hello'")
-
-    job_copy = job_.copy()
-    del job_copy["run"]["command-context"]
-    workerfn(config, job_copy, taskdesc)
-    assert_cmd("echo '{output}'")
-
-
 if __name__ == "__main__":
    main()
--- a/taskcluster/gecko_taskgraph/transforms/job/mach.py
+++ b/taskcluster/gecko_taskgraph/transforms/job/mach.py
@ -27,10 +27,6 @@ mach_schema = Schema(
        Required("comm-checkout"): bool,
        # Base work directory used to set up the task.
        Optional("workdir"): str,
-        # Context to substitute into the command using format string
-        # substitution (e.g {value}). This is useful if certain aspects of the
-        # command need to be generated in transforms.
-        Optional("command-context"): dict,
    }
 )

--- a/taskcluster/gecko_taskgraph/transforms/job/run_task.py
+++ b/taskcluster/gecko_taskgraph/transforms/job/run_task.py
@ -12,9 +12,8 @@ from mozbuild.util import memoize
 from mozpack import path
 from taskgraph.util.schema import Schema
 from taskgraph.util.yaml import load_yaml
-from voluptuous import Any, Extra, Optional, Required
+from voluptuous import Any, Optional, Required

-from gecko_taskgraph import GECKO
 from gecko_taskgraph.transforms.job import run_job_using
 from gecko_taskgraph.transforms.job.common import add_tooltool, support_vcs_checkout
 from gecko_taskgraph.transforms.task import taskref_or_string
@ -47,16 +46,6 @@ run_task_schema = Schema(
        # checkout arguments.  If a list, it will be passed directly; otherwise
        # it will be included in a single argument to `bash -cx`.
        Required("command"): Any([taskref_or_string], taskref_or_string),
-        # Context to substitute into the command using format string
-        # substitution (e.g {value}). This is useful if certain aspects of the
-        # command need to be generated in transforms.
-        Optional("command-context"): {
-            # If present, loads a set of context variables from an unnested yaml
-            # file. If a value is present in both the provided file and directly
-            # in command-context, the latter will take priority.
-            Optional("from-file"): str,
-            Extra: object,
-        },
        # Base work directory used to set up the task.
        Optional("workdir"): str,
        # If not false, tooltool downloads will be enabled via relengAPIProxy
@ -112,25 +101,6 @@ def script_url(config, script):
    return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"


-def substitute_command_context(command_context, command):
-    from_file = command_context.pop("from-file", None)
-    full_context = {}
-    if from_file:
-        full_context = load_yaml(os.path.join(GECKO, from_file))
-    else:
-        full_context = {}
-
-    full_context.update(command_context)
-
-    if isinstance(command, list):
-        for i in range(len(command)):
-            command[i] = command[i].format(**full_context)
-    else:
-        command = command.format(**full_context)
-
-    return command
-
-
@run_job_using(
    "docker-worker", "run-task", schema=run_task_schema, defaults=worker_defaults
 )
@ -154,12 +124,7 @@ def docker_worker_run_task(config, job, taskdesc):
            }
        )

-    if run.get("command-context"):
-        run_command = substitute_command_context(
-            run.get("command-context"), run["command"]
-        )
-    else:
-        run_command = run["command"]
+    run_command = run["command"]

    run_cwd = run.get("cwd")
    if run_cwd and run["checkout"]:
@ -263,11 +228,6 @@ def generic_worker_run_task(config, job, taskdesc):
                run_command = f'"{run_command}"'
        run_command = ["bash", "-cx", run_command]

-    if run.get("command-context"):
-        run_command = substitute_command_context(
-            run.get("command-context"), run_command
-        )
-
    if run["comm-checkout"]:
        command.append(
            "--comm-checkout={}/comm".format(taskdesc["worker"]["env"]["GECKO_PATH"])
--- a/taskcluster/gecko_taskgraph/transforms/source_test.py
+++ b/taskcluster/gecko_taskgraph/transforms/source_test.py
@ -254,8 +254,13 @@ def set_base_revision_in_tgdiff(config, jobs):
            yield job
            continue

-        job["run"]["command-context"] = {
-            "base_rev": data["changesets"][0]["parents"][0]
+        job["task-context"] = {
+            "from-object": {
+                "base_rev": data["changesets"][0]["parents"][0],
+            },
+            "substitution-fields": [
+                "run.command",
+            ],
        }
        yield job