Bug 1639211 - Implement --perfherder-app in mozperftest Perfherder layer. r=mozperftest-reviewers,tarek

This patch adds a new --perfherder-app that can be used to specify the shorthand name of the application being tested. The choices here are limited to what the perfherder schema can accept. We also add these settings to the existing taskcluster tasks to split the perfherder data. A couple tests are also added for this new flag.

Differential Revision: https://phabricator.services.mozilla.com/D76010

python/mozperftest/mozperftest/metrics/perfherder.py

@@ -31,6 +31,22 @@ class Perfherder(Layer):
             "default": "",
             "help": "Prefix the output files with this string.",
         },
+        "app": {
+            "type": str,
+            "default": "firefox",
+            "choices": [
+                "firefox",
+                "chrome-m",
+                "chrome",
+                "chromium",
+                "fennec",
+                "geckoview",
+                "fenix",
+                "refbrow",
+            ],
+            "help": "Shorthand name of application that is "
+                    "being tested (used in perfherder data).",
+        },
         "metrics": {
             "nargs": "*",
             "default": [],
@@ -81,6 +97,9 @@ class Perfherder(Layer):
             self.warning("No results left after filtering")
             return metadata
 
+        # XXX Add version info into this data
+        app_info = {"name": self.get_arg("app", default="firefox")}
+
         all_perfherder_data = None
         for name, res in results.items():
             settings = fullsettings[name]
@@ -103,7 +122,7 @@ class Perfherder(Layer):
                 name=name,
                 extra_options=settings.get("extraOptions"),
                 should_alert=settings.get("shouldAlert", False),
-                application=None,  # XXX No way to get application name and version
+                application=app_info,
                 alert_threshold=settings.get("alertThreshold", 2.0),
                 lower_is_better=settings.get("lowerIsBetter", True),
                 unit=settings.get("unit", "ms"),

python/mozperftest/mozperftest/tests/test_perfherder.py

@@ -152,5 +152,47 @@ def test_perfherder_exlude_stats():
     )
 
 
+def test_perfherder_app_name():
+    options = {
+        "perfherder": True,
+        "perfherder-prefix": "",
+        "perfherder-app": "fenix",
+        "perfherder-metrics": ["firstPaint"],
+    }
+
+    metrics, metadata, env = setup_env(options)
+
+    with temp_file() as output:
+        env.set_arg("output", output)
+        with metrics as m, silence():
+            m(metadata)
+        output_file = metadata.get_output()
+        with open(output_file) as f:
+            output = json.loads(f.read())
+
+    # Make sure that application setting is correct
+    assert output["application"]["name"] == "fenix"
+    assert "version" not in output["application"]
+
+
+def test_perfherder_bad_app_name():
+    options = {
+        "perfherder": True,
+        "perfherder-prefix": "",
+        "perfherder-app": "this is not an app",
+        "perfherder-metrics": ["firstPaint"],
+    }
+
+    metrics, metadata, env = setup_env(options)
+
+    # This will raise an error because the options method
+    # we use in tests skips the `choices` checks.
+    with pytest.raises(jsonschema.ValidationError):
+        with temp_file() as output:
+            env.set_arg("output", output)
+            with metrics as m, silence():
+                m(metadata)
+
+
 if __name__ == "__main__":
     mozunit.main()

taskcluster/ci/perftest/kind.yml

@@ -77,6 +77,7 @@ jobs:
                 --android-install-apk fenix_fennec_nightly_armeabi_v7a
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app fenix
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -100,6 +101,7 @@ jobs:
                 --android-activity org.mozilla.fenix.IntentReceiverActivity
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app fenix
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -123,6 +125,7 @@ jobs:
                 --android-install-apk gve_nightly_api16
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app geckoview
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -146,6 +149,7 @@ jobs:
                 --android-install-apk gve_nightly_api16
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app geckoview
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js