Bug 587344: run startup tests on signed Firefox builds. r=taskgraph-reviewers,aki

Depends on D107544

Differential Revision: https://phabricator.services.mozilla.com/D107545

taskcluster/ci/startup-test/kind.yml

@@ -0,0 +1,121 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    # We want to test the signed version of a build, to make sure
+    # any startup problems or crashes caused by signing are caught
+    # This means we depend on a different kind depending on the platform
+    # linux
+    - build-signing
+    # mac
+    - repackage
+    # windows
+    - repackage-signing
+
+transforms:
+    - taskgraph.transforms.startup_test:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    name: startup-test
+    description: Check whether or not a product crashes on startup
+    run-on-projects: ['mozilla-central']
+    worker:
+        max-run-time: 1200
+    run:
+        sparse-profile: mozharness
+    attributes:
+        build_type: opt
+    treeherder:
+        symbol: SUT
+        kind: test
+        tier: 3
+
+jobs:
+    linux32:
+        dependencies:
+            build-signing: build-signing-linux-shippable/opt
+        run:
+            using: run-task
+            cwd: "{checkout}"
+        extra-config:
+            upstream_kind: build-signing
+            upstream_artifact: target.tar.bz2
+            binary: firefox
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: ubuntu1804-test}
+        attributes:
+            build_platform: linux32
+        treeherder:
+            platform: linux-shippable/opt
+
+    linux64:
+        dependencies:
+            build-signing: build-signing-linux64-shippable/opt
+        run:
+            using: run-task
+            cwd: "{checkout}"
+        extra-config:
+            upstream_kind: build-signing
+            upstream_artifact: target.tar.bz2
+            binary: firefox
+        worker-type: b-linux
+        worker:
+            docker-image: {in-tree: ubuntu1804-test}
+        attributes:
+            build_platform: linux64
+        treeherder:
+            platform: linux64-shippable/opt
+
+    macosx64:
+        dependencies:
+            repackage: repackage-macosx64-shippable/opt
+        run:
+            using: mach
+            python-version: 3
+        extra-config:
+            upstream_kind: repackage
+            upstream_artifact: target.dmg
+            binary: Contents/MacOS/firefox
+        worker-type: t-osx-1014
+        attributes:
+            build_platform: macosx64
+        treeherder:
+            platform: macosx64-shippable/opt
+
+    win32:
+        dependencies:
+            repackage-signing: repackage-signing-win32-shippable/opt
+        run:
+            using: mach
+            python-version: 3
+        extra-config:
+            upstream_kind: repackage-signing
+            upstream_artifact: target.installer.exe
+            binary: core/firefox.exe
+        worker-type: t-win10-64-source
+        attributes:
+            build_platform: win32
+        treeherder:
+            platform: windows2012-32-shippable/opt
+
+    win64:
+        dependencies:
+            repackage-signing: repackage-signing-win64-shippable/opt
+        run:
+            using: mach
+            python-version: 3
+        extra-config:
+            upstream_kind: repackage-signing
+            upstream_artifact: target.installer.exe
+            binary: core/firefox.exe
+        worker-type: t-win10-64-source
+        attributes:
+            build_platform: win64
+        treeherder:
+            platform: windows2012-64-shippable/opt

taskcluster/docs/kinds.rst

@@ -725,3 +725,8 @@ fuzzing
 -------
 
 Performs fuzzing smoke tests
+
+startup-test
+------------
+
+Runs Firefox for a short period of time to see if it crashes

taskcluster/taskgraph/transforms/startup_test.py

@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def add_command(config, jobs):
+    for job in jobs:
+        extra_config = job.pop("extra-config")
+        upstream_kind = extra_config["upstream_kind"]
+        upstream_artifact = extra_config["upstream_artifact"]
+        binary = extra_config["binary"]
+        package_to_test = "<{}/public/build/{}>".format(
+            upstream_kind, upstream_artifact
+        )
+
+        if job["attributes"]["build_platform"].startswith("linux"):
+            job["run"]["command"] = {
+                "artifact-reference": ". $HOME/scripts/xvfb.sh && start_xvfb '1600x1200x24' 0 && "
+                + "python3 ./mach python testing/mozharness/scripts/does_it_crash.py "
+                + "--run-for 30 --thing-url "
+                + package_to_test
+                + " --thing-to-run "
+                + binary
+            }
+        else:
+            job["run"]["mach"] = {
+                "artifact-reference": "python testing/mozharness/scripts/does_it_crash.py "
+                + "--run-for 30 --thing-url "
+                + package_to_test
+                + " --thing-to-run "
+                + binary
+            }
+
+        yield job