Bug 1830741 - Add tests for mach try perf comparators. r=perftest-reviewers,AlexandruIonescu

This patch adds some unit tests for the comparators.

Depends on D177031

Differential Revision: https://phabricator.services.mozilla.com/D177201

tools/tryselect/test/python.ini

@@ -8,6 +8,7 @@ requirements = tools/tryselect/selectors/chooser/requirements.txt
 [test_fuzzy.py]
 [test_mozharness_integration.py]
 [test_perf.py]
+[test_perfcomparators.py]
 [test_presets.py]
 # Modifies "task_duration_history.json" in .mozbuild. Since other tests depend on this file, this test
 # shouldn't be run in parallel with those other tests.

tools/tryselect/test/test_perfcomparators.py

@@ -0,0 +1,135 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import tempfile
+from unittest import mock
+
+import mozunit
+import pytest
+from tryselect.selectors.perfselector.perfcomparators import (
+    BenchmarkComparator,
+    ComparatorNotFound,
+    get_comparator,
+)
+
+
+@pytest.mark.parametrize(
+    "test_link",
+    [
+        "https://github.com/mozilla-mobile/firefox-android/pull/1627",
+        "https://github.com/mozilla-mobile/firefox-android/pull/1876/"
+        "commits/17c7350cc37a4a85cea140a7ce54e9fd037b5365",
+    ],
+)
+def test_benchmark_comparator(test_link):
+    def _verify_extra_args(extra_args):
+        assert len(extra_args) == 3
+        if "commit" in test_link:
+            assert (
+                "benchmark-revision=17c7350cc37a4a85cea140a7ce54e9fd037b5365"
+                in extra_args
+            )
+        else:
+            assert "benchmark-revision=sha-for-link" in extra_args
+        assert "benchmark-repository=url-for-link" in extra_args
+        assert "benchmark-branch=ref-for-link" in extra_args
+
+    comparator = BenchmarkComparator(
+        None, None, None, [f"base-link={test_link}", f"new-link={test_link}"]
+    )
+
+    with mock.patch("requests.get") as mocked_get:
+        magic_get = mock.MagicMock()
+        magic_get.json.return_value = {
+            "head": {
+                "repo": {
+                    "html_url": "url-for-link",
+                },
+                "sha": "sha-for-link",
+                "ref": "ref-for-link",
+            }
+        }
+        magic_get.status_code = 200
+        mocked_get.return_value = magic_get
+
+        extra_args = []
+        comparator.setup_base_revision(extra_args)
+        _verify_extra_args(extra_args)
+
+        extra_args = []
+        comparator.setup_new_revision(extra_args)
+        _verify_extra_args(extra_args)
+
+
+def test_benchmark_comparator_no_pr_links():
+    def _verify_extra_args(extra_args):
+        assert len(extra_args) == 3
+        assert "benchmark-revision=rev" in extra_args
+        assert "benchmark-repository=link" in extra_args
+        assert "benchmark-branch=fake" in extra_args
+
+    comparator = BenchmarkComparator(
+        None,
+        None,
+        None,
+        [
+            "base-repo=link",
+            "base-branch=fake",
+            "base-revision=rev",
+            "new-repo=link",
+            "new-branch=fake",
+            "new-revision=rev",
+        ],
+    )
+
+    with mock.patch("requests.get") as mocked_get:
+        magic_get = mock.MagicMock()
+        magic_get.json.return_value = {
+            "head": {
+                "repo": {
+                    "html_url": "url-for-link",
+                },
+                "sha": "sha-for-link",
+                "ref": "ref-for-link",
+            }
+        }
+        magic_get.status_code = 200
+        mocked_get.return_value = magic_get
+
+        extra_args = []
+        comparator.setup_base_revision(extra_args)
+        _verify_extra_args(extra_args)
+
+        extra_args = []
+        comparator.setup_new_revision(extra_args)
+        _verify_extra_args(extra_args)
+
+
+def test_get_comparator_bad_name():
+    with pytest.raises(ComparatorNotFound):
+        get_comparator("BadName")
+
+
+def test_get_comparator_bad_script():
+    with pytest.raises(ComparatorNotFound):
+        with tempfile.NamedTemporaryFile() as tmpf:
+            tmpf.close()
+            get_comparator(tmpf.name)
+
+
+def test_get_comparator_benchmark_name():
+    comparator_klass = get_comparator("BenchmarkComparator")
+    assert comparator_klass.__name__ == "BenchmarkComparator"
+
+
+def test_get_comparator_benchmark_script():
+    # If the get_comparator method is working for scripts, then
+    # it should find the first defined class in this file, or the
+    # first imported class that matches it
+    comparator_klass = get_comparator(__file__)
+    assert comparator_klass.__name__ == "BenchmarkComparator"
+
+
+if __name__ == "__main__":
+    mozunit.main()