Bug 1577110: Switch geckodriver signing to use toolchain tasks r=tomprince

Differential Revision: https://phabricator.services.mozilla.com/D72754

taskcluster/ci/config.yml

@@ -157,6 +157,7 @@ index:
         - 'devedition'
         - 'source'
         - 'system-symbols'
+        - 'geckodriver'
 
 try:
     # We have a few platforms for which we want to do some "extra" builds, or at

taskcluster/ci/geckodriver-repack/kind.yml

@@ -1,128 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-loader: taskgraph.loader.transform:loader
-
-kind-dependencies:
-    - build
-    - fetch
-
-transforms:
-    - taskgraph.transforms.build_attrs:transforms
-    - taskgraph.transforms.job:transforms
-    - taskgraph.transforms.task:transforms
-
-job-defaults:
-    attributes:
-        nightly: true
-    description: "Extract geckodriver binary from common tests package"
-    treeherder:
-        kind: other
-        symbol: Gd(repack)
-        tier: 2
-    worker-type: b-linux
-    worker:
-        max-run-time: 1200
-        docker-image: {in-tree: debian9-amd64-build}
-        chain-of-trust: true
-    run:
-        using: run-task
-        checkout: false
-
-jobs:
-    linux-nightly/opt:
-        treeherder:
-            platform: linux32/opt
-        dependencies:
-            build-linux/opt: build-linux-shippable/opt
-        fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
-        worker:
-            artifacts:
-                - type: file
-                  name: public/geckodriver.tar.gz
-                  path: /builds/worker/geckodriver.tar.gz
-        run-on-projects: ['try', 'release']  # Do not run on autoland, Bug 1599197
-        run:
-            command: >
-                cd /builds/worker/fetches/bin/ &&
-                tar -cvzf geckodriver.tar.gz geckodriver &&
-                cp geckodriver.tar.gz /builds/worker
-
-    linux64-nightly/opt:
-        treeherder:
-            platform: linux64/opt
-        dependencies:
-            build-linux64/opt: build-linux64-shippable/opt
-        fetches:
-            build-linux64/opt:
-                - artifact: target.common.tests.tar.gz
-        worker:
-            artifacts:
-                - type: file
-                  name: public/geckodriver.tar.gz
-                  path: /builds/worker/geckodriver.tar.gz
-        run:
-            command: >
-                cd /builds/worker/fetches/bin/ &&
-                tar -cvzf geckodriver.tar.gz geckodriver &&
-                cp geckodriver.tar.gz /builds/worker
-
-    macosx64-nightly/opt:
-        treeherder:
-            platform: osx-cross/opt
-        dependencies:
-            build-macosx64/opt: build-macosx64-shippable/opt
-        fetches:
-            build-macosx64/opt:
-                - artifact: target.common.tests.tar.gz
-        worker:
-            artifacts:
-                - type: file
-                  name: public/geckodriver.tar.gz
-                  path: /builds/worker/geckodriver.tar.gz
-        run:
-            command: >
-                cd /builds/worker/fetches/bin/ &&
-                tar -cvzf geckodriver.tar.gz geckodriver &&
-                cp geckodriver.tar.gz /builds/worker
-
-    win32-nightly/opt:
-        treeherder:
-            platform: windows2012-32/opt
-        dependencies:
-            build-win32/opt: build-win32-shippable/opt
-        fetches:
-            build-win32/opt:
-                - artifact: target.common.tests.tar.gz
-        worker:
-            artifacts:
-                - type: file
-                  name: public/geckodriver.zip
-                  path: /builds/worker/geckodriver.zip
-        run:
-            command: >
-                cd /builds/worker/fetches/bin/ &&
-                zip -r geckodriver.zip geckodriver.exe &&
-                cp geckodriver.zip /builds/worker
-
-    win64-nightly/opt:
-        treeherder:
-            platform: windows2012-64/opt
-        dependencies:
-            build-win64/opt: build-win64-shippable/opt
-        fetches:
-            build-win64/opt:
-                - artifact: target.common.tests.tar.gz
-        worker:
-            artifacts:
-                - type: file
-                  name: public/geckodriver.zip
-                  path: /builds/worker/geckodriver.zip
-        run:
-            command: >
-                cd /builds/worker/fetches/bin/ &&
-                zip -r geckodriver.zip geckodriver.exe &&
-                cp geckodriver.zip /builds/worker

taskcluster/ci/geckodriver-signing/kind.yml

@@ -10,7 +10,7 @@ transforms:
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
-    - geckodriver-repack
+    - toolchain
 
-job-template:
+only-for-attributes:
-    shipping-phase: promote
+    - geckodriver

taskcluster/ci/toolchain/geckodriver.yml

@@ -7,10 +7,17 @@ job-defaults:
     worker-type: b-linux
     worker:
         max-run-time: 1800
+        artifacts:
+            - name: public
+              type: directory
+              path: /builds/worker/artifacts
     run-on-projects: ['trunk']
+    attributes:
+        build_type: opt
+        geckodriver: true
     run:
         script: build-geckodriver.sh
-        toolchain-artifact: public/build/geckodriver.tar.xz
+        toolchain-artifact: public/geckodriver.tar.gz
         sparse-profile: null
         resources:
             - 'testing/geckodriver'
@@ -21,26 +28,32 @@ job-defaults:
 
 linux32-geckodriver:
     treeherder:
-        symbol: TL32(geckodriver)
+        symbol: TL32(gd)
+        platform: linux32/opt
     run:
         arguments: ['i686-unknown-linux-gnu']
-    run-on-projects: ['try', 'release']  # Do not run on autoland - Bug 1599197
+    attributes:
+        build_platform: linux-geckodriver
     fetches:
         toolchain:
             - linux64-rust
 
 linux64-geckodriver:
     treeherder:
-        symbol: TL(geckodriver)
+        symbol: TL(gd)
+        platform: linux64/opt
     run:
         arguments: ['x86_64-unknown-linux-gnu']
+    attributes:
+        build_platform: linux64-geckodriver
     fetches:
         toolchain:
             - linux64-rust
 
 macosx64-geckodriver:
     treeherder:
-        symbol: TM(geckodriver)
+        symbol: TM(gd)
+        platform: macosx64/opt
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
@@ -49,6 +62,9 @@ macosx64-geckodriver:
         resources:
             - taskcluster/scripts/misc/tooltool-download.sh
         tooltool-downloads: internal
+        toolchain-artifact: public/geckodriver.tar.gz
+    attributes:
+        build_platform: macosx64-geckodriver
     fetches:
         toolchain:
             - linux64-cctools-port
@@ -58,7 +74,8 @@ macosx64-geckodriver:
 
 win32-geckodriver:
     treeherder:
-        symbol: TW32(geckodriver)
+        symbol: TW32(gd)
+        platform: win32/opt
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
@@ -66,8 +83,10 @@ win32-geckodriver:
         arguments: ['i686-pc-windows-msvc']
         resources:
             - taskcluster/scripts/misc/tooltool-download.sh
-        toolchain-artifact: public/build/geckodriver.tar.bz2
+        toolchain-artifact: public/geckodriver.zip
         tooltool-downloads: internal
+    attributes:
+        build_platform: win32-geckodriver
     fetches:
         toolchain:
             - linux64-clang
@@ -75,7 +94,8 @@ win32-geckodriver:
 
 win64-geckodriver:
     treeherder:
-        symbol: TW64(geckodriver)
+        symbol: TW64(gd)
+        platform: win64/opt
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
@@ -83,8 +103,10 @@ win64-geckodriver:
         arguments: ['x86_64-pc-windows-msvc']
         resources:
             - taskcluster/scripts/misc/tooltool-download.sh
-        toolchain-artifact: public/build/geckodriver.tar.bz2
+        toolchain-artifact: public/geckodriver.zip
         tooltool-downloads: internal
+    attributes:
+        build_platform: win64-geckodriver
     fetches:
         toolchain:
             - linux64-clang

taskcluster/docs/attributes.rst

@@ -352,3 +352,8 @@ If set to true, this task will not be checked to see that
 MOZ_AUTOMATION_PACKAGE_TESTS is set correctly based on whether or not the task
 has dependent tests. This should only be used in very unique situations, such
 as Windows AArch64 builds that copy test packages between build tasks.
+
+geckodriver
+===========
+If non-empty, declares that the (toolchain) task is a `geckodriver`
+task that produces a binary that should be signed.

taskcluster/docs/kinds.rst

@@ -680,11 +680,6 @@ Tasks that take a build configured for PGO and run the binary against a sample
 set to generate profile data. This is the 2nd stage of the full 3-step PGO
 process.
 
-geckodriver-repack
-------------------
-Tasks to repackage the geckodriver binary from a build tasks's common
-test archive into it's own archive.
-
 geckodriver-signing
 -------------------
 Signing for geckodriver binary.

taskcluster/scripts/misc/build-geckodriver.sh

@@ -14,12 +14,12 @@ if [ -n "$TOOLTOOL_MANIFEST" ]; then
 fi
 
 EXE=
-COMPRESS_EXT=xz
+COMPRESS_EXT=gz
 
 case "$TARGET" in
 *windows-msvc)
   EXE=.exe
-  COMPRESS_EXT=bz2
+  COMPRESS_EXT=zip
   if [[ $TARGET == "i686-pc-windows-msvc" ]]; then
     . $GECKO_PATH/taskcluster/scripts/misc/vs-setup32.sh
     export CARGO_TARGET_I686_PC_WINDOWS_MSVC_LINKER=$MOZ_FETCHES_DIR/clang/bin/lld-link
@@ -52,10 +52,15 @@ cp $GECKO_PATH/.cargo/config.in $GECKO_PATH/.cargo/config
 cargo build --frozen --verbose --release --target "$TARGET"
 
 cd $GECKO_PATH
+mkdir -p $UPLOAD_DIR
 
 cp target/$TARGET/release/geckodriver$EXE .
-tar -acf geckodriver.tar.$COMPRESS_EXT geckodriver$EXE
+if [ "$COMPRESS_EXT" = "zip" ]; then
-mkdir -p $UPLOAD_DIR
+    zip geckodriver.zip geckodriver$EXE
-cp geckodriver.tar.$COMPRESS_EXT $UPLOAD_DIR
+    cp geckodriver.zip $UPLOAD_DIR
+else
+    tar -acf geckodriver.tar.$COMPRESS_EXT geckodriver$EXE
+    cp geckodriver.tar.$COMPRESS_EXT $UPLOAD_DIR
+fi
 
 . $GECKO_PATH/taskcluster/scripts/misc/vs-cleanup.sh

taskcluster/taskgraph/transforms/geckodriver_signing.py

@@ -18,7 +18,7 @@ from taskgraph.transforms.task import task_description_schema
 from voluptuous import Required, Optional
 
 repackage_signing_description_schema = schema.extend({
-    Required('depname', default='geckodriver-repackage'): text_type,
+    Required('depname', default='toolchain-geckodriver'): text_type,
     Optional('label'): text_type,
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
@@ -29,19 +29,18 @@ transforms.add_validate(repackage_signing_description_schema)
 
 
 @transforms.add
-def make_repackage_signing_description(config, jobs):
+def make_signing_description(config, jobs):
     for job in jobs:
         dep_job = job['primary-dependency']
+
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['repackage_type'] = 'repackage-signing'
 
         treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'Gd(s)')
+        dep_treeherder = dep_job.task.get('extra', {}).get('treeherder', {})
+        treeherder.setdefault('symbol', '{}(gd-s)'.format(dep_treeherder['groupSymbol']))
         treeherder.setdefault('platform', dep_job.task.get('extra', {}).get('treeherder-platform'))
-        treeherder.setdefault(
+        treeherder.setdefault('tier', dep_treeherder.get('tier', 1))
-            'tier',
-            dep_job.task.get('extra', {}).get('treeherder', {}).get('tier', 1)
-        )
         treeherder.setdefault('kind', 'build')
 
         dependencies = {dep_job.kind: dep_job.label}
@@ -51,9 +50,8 @@ def make_repackage_signing_description(config, jobs):
             if k != 'docker-image'
         })
 
-        description = "Signing Geckodriver for build '{}/{}'".format(
+        description = "Signing Geckodriver for build '{}'".format(
             attributes.get('build_platform'),
-            attributes.get('build_type'),
         )
 
         build_platform = dep_job.attributes.get('build_platform')
@@ -62,10 +60,12 @@ def make_repackage_signing_description(config, jobs):
             build_platform, is_nightly, config
         )
 
-        upstream_artifacts = _craft_upstream_artifacts(dep_job.kind, build_platform)
+        upstream_artifacts = _craft_upstream_artifacts(dep_job, dep_job.kind, build_platform)
 
         scopes = [signing_cert_scope]
 
+        platform = build_platform.split("-")[0]
+
         task = {
             'label': job['label'],
             'description': description,
@@ -77,8 +77,9 @@ def make_repackage_signing_description(config, jobs):
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
-            'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
+            'run-on-projects': ['mozilla-central'],
+            'index': {"product": "geckodriver", "job-name": platform},
         }
 
         if build_platform.startswith('macosx'):
@@ -102,22 +103,19 @@ def make_repackage_signing_description(config, jobs):
         yield task
 
 
-def _craft_upstream_artifacts(dependency_kind, build_platform):
+def _craft_upstream_artifacts(dep_job, dependency_kind, build_platform):
     if build_platform.startswith('win'):
         signing_format = 'autograph_authenticode'
-        extension = 'zip'
     elif build_platform.startswith('linux'):
         signing_format = 'autograph_gpg'
-        extension = 'tar.gz'
     elif build_platform.startswith('macosx'):
         signing_format = 'mac_geckodriver'
-        extension = 'tar.gz'
     else:
         raise ValueError('Unsupported build platform "{}"'.format(build_platform))
 
     return [{
         'taskId': {'task-reference': '<{}>'.format(dependency_kind)},
-        'taskType': 'repackage',
+        'taskType': 'build',
-        'paths': ['public/geckodriver.{}'.format(extension)],
+        'paths': [dep_job.attributes['toolchain-artifact']],
         'formats': [signing_format],
     }]