Bug 1631839 - Update mobile routes to use new namespace r=tarek,sparky,acreskey

Differential Revision: https://phabricator.services.mozilla.com/D81038

python/mozperftest/mozperftest/system/android.py

@@ -14,17 +14,16 @@ from mozperftest.utils import download_file
 
 _ROOT_URL = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/"
 _FENIX_FENNEC_BUILDS = (
-    "project.mobile.fenix.v2.fennec-nightly.latest/artifacts/public/build/"
+    "mobile.v2.fenix.fennec-nightly.latest.{architecture}"
+    "/artifacts/public/build/{architecture}/geckoNightly/target.apk"
 )
 _GV_BUILDS = "gecko.v2.mozilla-central.nightly.latest.mobile.android-"
 
 _PERMALINKS = {
     "fenix_fennec_nightly_armeabi_v7a": _ROOT_URL
-    + _FENIX_FENNEC_BUILDS
-    + "armeabi-v7a/geckoNightly/target.apk",
+    + _FENIX_FENNEC_BUILDS.format(architecture="armeabi-v7a"),
     "fenix_fennec_nightly_arm64_v8a": _ROOT_URL
-    + _FENIX_FENNEC_BUILDS
-    + "arm64-v8a/geckoNightly/target.apk",
+    + _FENIX_FENNEC_BUILDS.format(architecture="arm64-v8a"),
     # The two following aliases are used for Fenix multi-commit testing in CI
     "fenix_nightlysim_multicommit_arm64_v8a": None,
     "fenix_nightlysim_multicommit_armeabi_v7a": None,

python/mozperftest/mozperftest/tests/test_utils.py

@@ -17,6 +17,7 @@ from mozperftest.utils import (
     install_package,
     build_test_list,
     get_multi_tasks_url,
+    get_revision_namespace_url,
     convert_day,
     load_class,
 )
@@ -119,22 +120,31 @@ def test_convert_day():
     assert convert_day("yesterday") == yesterday.strftime("%Y.%m.%d")
 
 
+def test_revision_namespace_url():
+    route = "FakeBuildRoute"
+    day = "2020.06.08"
+    buildurl = get_revision_namespace_url(route, day=day)
+    assert day in buildurl and route in buildurl
+    assert buildurl.endswith(".revision")
+
+
 def test_multibuild_url():
     route = "FakeBuildRoute"
     day = "2020.06.08"
-    buildurl = get_multi_tasks_url(route, day=day)
-    assert day in buildurl and route in buildurl
+    revision = "deadbeef"
+    buildurl = get_multi_tasks_url(route, revision, day=day)
+    assert all(item in buildurl for item in (route, day, revision))
 
     with mock.patch("mozperftest.utils.date") as mockeddate:
         mockeddate.today.return_value = mockeddate
         mockeddate.strftime.return_value = "2020.07.09"
-        buildurl = get_multi_tasks_url(route, day="today")
+        buildurl = get_multi_tasks_url(route, revision, day="today")
         assert "2020.07.09" in buildurl and route in buildurl
 
         with mock.patch("mozperftest.utils.timedelta"):
             mockeddate.__sub__.return_value = mockeddate
             mockeddate.strftime.return_value = "2020.08.09"
-            buildurl = get_multi_tasks_url(route)
+            buildurl = get_multi_tasks_url(route, revision)
             assert "2020.08.09" in buildurl and route in buildurl
 
 

python/mozperftest/mozperftest/utils.py

@@ -18,7 +18,9 @@ import importlib
 
 
 RETRY_SLEEP = 10
-MULTI_TASK_ROOT = "https://firefox-ci-tc.services.mozilla.com/api/index/v1/tasks/"
+API_ROOT = "https://firefox-ci-tc.services.mozilla.com/api/index/v1"
+MULTI_REVISION_ROOT = f"{API_ROOT}/namespaces"
+MULTI_TASK_ROOT = f"{API_ROOT}/tasks"
 
 
 @contextlib.contextmanager
@@ -238,14 +240,21 @@ def convert_day(day):
     return day
 
 
-def get_multi_tasks_url(route, day="yesterday"):
+def get_revision_namespace_url(route, day="yesterday"):
+    """Builds a URL to obtain all the namespaces of a given build route for a single day.
+    """
+    day = convert_day(day)
+    return f"""{MULTI_REVISION_ROOT}/{route}.{day}.revision"""
+
+
+def get_multi_tasks_url(route, revision, day="yesterday"):
     """Builds a URL to obtain all the tasks of a given build route for a single day.
 
     If previous is true, then we get builds from the previous day,
     otherwise, we look at the current day.
     """
     day = convert_day(day)
-    return f"""{MULTI_TASK_ROOT}{route}.{day}.revision"""
+    return f"""{MULTI_TASK_ROOT}/{route}.{day}.revision.{revision}"""
 
 
 def strtobool(val):

taskcluster/ci/condprof/kind.yml

@@ -173,7 +173,7 @@ jobs:
         run:
             run-as-root: true
             command: >-
-                curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.nightly.latest/artifacts/public/build/armeabi-v7a/geckoNightly/target.apk &&
+                curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/mobile.v2.fenix.nightly.latest.armeabi-v7a/artifacts/public/build/armeabi-v7a/geckoNightly/target.apk &&
                 adb install -r target.apk &&
                 rm -rf target.apk &&
                 virtualenv/virtualenv.py -p python3 . &&
@@ -240,7 +240,7 @@ jobs:
         run:
             run-as-root: true
             command: >-
-                curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.nightly.latest/artifacts/public/build/arm64-v8a/geckoNightly/target.apk &&
+                curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/mobile.v2.fenix.nightly.latest.arm64-v8a/artifacts/public/build/arm64-v8a/geckoNightly/target.apk &&
                 adb install -r target.apk &&
                 rm -rf target.apk &&
                 virtualenv/virtualenv.py -p python3 . &&

taskcluster/ci/test/browsertime-mobile.yml

@@ -22,10 +22,10 @@ job-defaults:
             fenix:
                 by-test-platform:
                     android-hw.*-aarch64.*/.*:
-                        index: project.mobile.fenix.v2.nightly.latest
+                        index: mobile.v2.fenix.nightly.latest.arm64-v8a
                         name: build/arm64-v8a/geckoNightly/target.apk
                     default:
-                        index: project.mobile.fenix.v2.nightly.latest
+                        index: mobile.v2.fenix.nightly.latest.armeabi-v7a
                         name: build/armeabi-v7a/geckoNightly/target.apk
             fennec:
                 by-test-platform:
@@ -38,10 +38,10 @@ job-defaults:
             refbrow:
                 by-test-platform:
                     android-hw.*-aarch64.*/.*:
-                        index: project.mobile.reference-browser.v3.raptor.latest
+                        index: mobile.v2.reference-browser.raptor.latest.arm64-v8a
                         name: target.arm64-v8a.apk
                     default:
-                        index: project.mobile.reference-browser.v3.raptor.latest
+                        index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a
                         name: target.armeabi-v7a.apk
             default: geckoview_example.apk
     e10s: true

taskcluster/ci/test/raptor-fenix.yml

@@ -11,10 +11,10 @@ job-defaults:
     target:
         by-test-platform:
             android-hw.*-aarch64.*/.*:
-                index: project.mobile.fenix.v2.nightly.latest
+                index: mobile.v2.fenix.nightly.latest.arm64-v8a
                 name: build/arm64-v8a/geckoNightly/target.apk
             default:
-                index: project.mobile.fenix.v2.nightly.latest
+                index: mobile.v2.fenix.nightly.latest.armeabi-v7a
                 name: build/armeabi-v7a/geckoNightly/target.apk
     e10s: true
     virtualization: hardware

taskcluster/ci/test/raptor-refbrow.yml

@@ -11,10 +11,10 @@ job-defaults:
     target:
         by-test-platform:
             android-hw.*-aarch64.*/.*:
-                index: project.mobile.reference-browser.v3.raptor.latest
+                index: mobile.v2.reference-browser.raptor.latest.arm64-v8a
                 name: target.arm64-v8a.apk
             default:
-                index: project.mobile.reference-browser.v3.raptor.latest
+                index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a
                 name: target.armeabi-v7a.apk
     virtualization: hardware
     mozharness:

testing/performance/hooks_applink.py

@@ -7,7 +7,7 @@ import pathlib
 
 from mozperftest.test.browsertime import add_options
 from mozperftest.system.android import _ROOT_URL
-from mozperftest.utils import download_file, get_multi_tasks_url
+from mozperftest.utils import download_file, get_multi_tasks_url, get_revision_namespace_url
 
 URL = "'https://www.example.com'"
 
@@ -18,9 +18,8 @@ COMMON_OPTIONS = [("processStartTime", "true"),
                   ("firefox.android.intentArgument", "'-d'"),
                   ("firefox.android.intentArgument", URL)]
 
-NIGHTLY_SIM_ROUTE = "project.mobile.fenix.v2.nightly-simulation"
-G5_SUFFIX = "artifacts/public/build/armeabi-v7a/geckoNightly/target.apk"
-P2_SUFFIX = "artifacts/public/build/arm64-v8a/geckoNightly/target.apk"
+NIGHTLY_SIM_ROUTE = "mobile.v2.fenix.nightly-simulation"
+ROUTE_SUFFIX = "artifacts/public/build/{architecture}/geckoNightly/target.apk"
 
 build_generator = None
 
@@ -32,31 +31,46 @@ def before_iterations(kw):
         return
 
     # Get the builds to test
-    build_url = get_multi_tasks_url(NIGHTLY_SIM_ROUTE, day=kw["test_date"])
-    tmpfile = pathlib.Path(tempfile.mkdtemp(), "alltasks.json")
+    architecture = "arm64-v8a" if "arm64_v8a" in kw.get("android_install_apk") else "armeabi-v7a"
+    json_ = _fetch_json(get_revision_namespace_url, NIGHTLY_SIM_ROUTE, day=kw["test_date"])
+    namespaces = json_["namespaces"]
+    revisions = [namespace["name"] for namespace in namespaces]
+
+    tasks = []
+    for revision in revisions:
+        json_ = _fetch_json(get_multi_tasks_url, NIGHTLY_SIM_ROUTE, revision, day=kw["test_date"])
+        for task in json_["tasks"]:
+            route = task["namespace"]
+            task_architecture = route.split(".")[-1]
+            if task_architecture == architecture:
+                tasks.append({
+                    "revision": revision,
+                    "route": route,
+                    "route_suffix": ROUTE_SUFFIX.format(architecture=task_architecture),
+                })
+
+    kw["test_iterations"] = len(tasks)
+
+    def _build_iterator():
+        for task in tasks:
+            revision = task["revision"]
+            print(f"Testing revision {revision}")
+            download_url = f'{_ROOT_URL}{task["route"]}/{task["route_suffix"]}'
+            yield revision, [download_url]
+
+    build_generator = _build_iterator()
+
+    return kw
+
+
+def _fetch_json(get_url_function, *args, **kwargs):
+    build_url = get_url_function(*args, **kwargs)
+    tmpfile = pathlib.Path(tempfile.mkdtemp(), "temp.json")
     download_file(build_url, tmpfile)
 
     # Set the number of test-iterations to the number of builds
     with tmpfile.open() as f:
-        tasks = json.load(f)["tasks"]
-    kw["test_iterations"] = len(tasks)
-
-    # Finally, make an iterator for the builds (used in `before_runs`)
-    route_suffix = G5_SUFFIX
-    if "arm64_v8a" in kw.get("android_install_apk"):
-        route_suffix = P2_SUFFIX
-
-    def _build_iterator(route_suffix):
-        for task in tasks:
-            route = task["namespace"]
-            revision = route.split(".")[-1]
-            print("Testing revision %s" % revision)
-            download_url = "%s%s/%s" % (_ROOT_URL, route, route_suffix)
-            yield revision, [download_url]
-
-    build_generator = _build_iterator(route_suffix)
-
-    return kw
+        return json.load(f)
 
 
 def before_runs(env, **kw):