From 7788d4b5760c27e294fcb79e98e696b5028bdc8f Mon Sep 17 00:00:00 2001
From: Gregory Mierzwinski <gmierz2@outlook.com>
Date: Tue, 19 May 2020 19:07:27 +0000
Subject: [PATCH] Bug 1639211 - Implement --perfherder-app in mozperftest
 Perfherder layer. r=mozperftest-reviewers,tarek

This patch adds a new --perfherder-app that can be used to specify the shorthand name of the application being tested. The choices here are limited to what the perfherder schema can accept. We also add these settings to the existing taskcluster tasks to split the perfherder data. A couple tests are also added for this new flag.

Differential Revision: https://phabricator.services.mozilla.com/D76010
---
 .../mozperftest/metrics/perfherder.py         | 21 +++++++++-
 .../mozperftest/tests/test_perfherder.py      | 42 +++++++++++++++++++
 taskcluster/ci/perftest/kind.yml              |  4 ++
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/python/mozperftest/mozperftest/metrics/perfherder.py b/python/mozperftest/mozperftest/metrics/perfherder.py
index 606f2dbf1188..83f6effc8259 100644
--- a/python/mozperftest/mozperftest/metrics/perfherder.py
+++ b/python/mozperftest/mozperftest/metrics/perfherder.py
@@ -31,6 +31,22 @@ class Perfherder(Layer):
             "default": "",
             "help": "Prefix the output files with this string.",
         },
+        "app": {
+            "type": str,
+            "default": "firefox",
+            "choices": [
+                "firefox",
+                "chrome-m",
+                "chrome",
+                "chromium",
+                "fennec",
+                "geckoview",
+                "fenix",
+                "refbrow",
+            ],
+            "help": "Shorthand name of application that is "
+                    "being tested (used in perfherder data).",
+        },
         "metrics": {
             "nargs": "*",
             "default": [],
@@ -81,6 +97,9 @@ class Perfherder(Layer):
             self.warning("No results left after filtering")
             return metadata
 
+        # XXX Add version info into this data
+        app_info = {"name": self.get_arg("app", default="firefox")}
+
         all_perfherder_data = None
         for name, res in results.items():
             settings = fullsettings[name]
@@ -103,7 +122,7 @@ class Perfherder(Layer):
                 name=name,
                 extra_options=settings.get("extraOptions"),
                 should_alert=settings.get("shouldAlert", False),
-                application=None,  # XXX No way to get application name and version
+                application=app_info,
                 alert_threshold=settings.get("alertThreshold", 2.0),
                 lower_is_better=settings.get("lowerIsBetter", True),
                 unit=settings.get("unit", "ms"),
diff --git a/python/mozperftest/mozperftest/tests/test_perfherder.py b/python/mozperftest/mozperftest/tests/test_perfherder.py
index 377ff6a34dd3..e7072f81dd43 100644
--- a/python/mozperftest/mozperftest/tests/test_perfherder.py
+++ b/python/mozperftest/mozperftest/tests/test_perfherder.py
@@ -152,5 +152,47 @@ def test_perfherder_exlude_stats():
     )
 
 
+def test_perfherder_app_name():
+    options = {
+        "perfherder": True,
+        "perfherder-prefix": "",
+        "perfherder-app": "fenix",
+        "perfherder-metrics": ["firstPaint"],
+    }
+
+    metrics, metadata, env = setup_env(options)
+
+    with temp_file() as output:
+        env.set_arg("output", output)
+        with metrics as m, silence():
+            m(metadata)
+        output_file = metadata.get_output()
+        with open(output_file) as f:
+            output = json.loads(f.read())
+
+    # Make sure that application setting is correct
+    assert output["application"]["name"] == "fenix"
+    assert "version" not in output["application"]
+
+
+def test_perfherder_bad_app_name():
+    options = {
+        "perfherder": True,
+        "perfherder-prefix": "",
+        "perfherder-app": "this is not an app",
+        "perfherder-metrics": ["firstPaint"],
+    }
+
+    metrics, metadata, env = setup_env(options)
+
+    # This will raise an error because the options method
+    # we use in tests skips the `choices` checks.
+    with pytest.raises(jsonschema.ValidationError):
+        with temp_file() as output:
+            env.set_arg("output", output)
+            with metrics as m, silence():
+                m(metadata)
+
+
 if __name__ == "__main__":
     mozunit.main()
diff --git a/taskcluster/ci/perftest/kind.yml b/taskcluster/ci/perftest/kind.yml
index 30ee76e1dd9d..e0087411a187 100644
--- a/taskcluster/ci/perftest/kind.yml
+++ b/taskcluster/ci/perftest/kind.yml
@@ -77,6 +77,7 @@ jobs:
                 --android-install-apk fenix_fennec_nightly_armeabi_v7a
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app fenix
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -100,6 +101,7 @@ jobs:
                 --android-activity org.mozilla.fenix.IntentReceiverActivity
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app fenix
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -123,6 +125,7 @@ jobs:
                 --android-install-apk gve_nightly_api16
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app geckoview
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js
@@ -146,6 +149,7 @@ jobs:
                 --android-install-apk gve_nightly_api16
                 --hooks testing/performance/hooks_applink.py
                 --perfherder
+                --perfherder-app geckoview
                 --browsertime-geckodriver ${MOZ_FETCHES_DIR}/geckodriver
                 --output $MOZ_FETCHES_DIR/../artifacts
                 testing/performance/perftest_applink.js