mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 1884930 - Use per-commit fenix and geckoview builds instead of downloading daily build. r=perftest-reviewers,taskgraph-reviewers,sparky,bhearsum 

As a result of the mono-repo change we have a very easy way to run the startup tests on each commit vs on a nightly build.
This change allows us to push a try job, build the fenix/focus/geckoview_example apk and test the startup impact!

Differential Revision: https://phabricator.services.mozilla.com/D208240
This commit is contained in:
andrej 2024-05-13 16:50:03 +00:00
Родитель 13de78eab7
Коммит 34cd136d5c
7 изменённых файлов: 204 добавлений и 205 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

98
python/mozperftest/mozperftest/system/android_startup.py
Просмотреть файл

@ -1,13 +1,17 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import pathlib
import re
import statistics
import time
from datetime import datetime, timedelta
from datetime import datetime
import mozdevice
from mozperftest.utils import ON_TRY
from .android import AndroidDevice
DATETIME_FORMAT = "%Y.%m.%d"
@ -19,13 +23,6 @@ PROD_GVEX = "geckoview_example"
PROD_CHRM = "chrome-m"
MOZILLA_PRODUCTS = [PROD_FENIX, PROD_FOCUS, PROD_GVEX]
KEY_NAME = "name"
KEY_PRODUCT = "product"
KEY_DATETIME = "date"
KEY_COMMIT = "commit"
KEY_ARCHITECTURE = "architecture"
KEY_TEST_NAME = "test_name"
MEASUREMENT_DATA = ["mean", "median", "standard_deviation"]
OLD_VERSION_FOCUS_PAGE_START_LINE_COUNT = 3
NEW_VERSION_FOCUS_PAGE_START_LINE_COUNT = 2
@ -99,8 +96,10 @@ PROD_TO_CHANNEL_TO_PKGID = {
},
PROD_GVEX: {
"nightly": "org.mozilla.geckoview_example",
"release": "org.mozilla.geckoview_example",
},
PROD_CHRM: {
"nightly": "com.android.chrome",
"release": "com.android.chrome",
},
}
@ -167,22 +166,12 @@ class AndroidStartUp(AndroidDevice):
"test-name": {
"type": str,
"default": "",
"help": "This is the startup android test that will be run on the a51",
},
"apk_metadata": {
"type": str,
"default": "",
"help": "This is the startup android test that will be run on the a51",
"help": "This is the startup android test that will be run on the mobile device",
},
"product": {
"type": str,
"default": "",
"help": "This is the startup android test that will be run on the a51",
},
"release-channel": {
"type": str,
"default": "",
"help": "This is the startup android test that will be run on the a51",
"help": "This is the product that you are testing the startup of(ex. chrome, fenix, focus)",
},
}
@ -195,9 +184,9 @@ class AndroidStartUp(AndroidDevice):
def run(self, metadata):
options = metadata.script["options"]
self.test_name = self.get_arg("test-name")
self.apk_metadata = self.get_arg("apk-metadata")
self.product = self.get_arg("product")
self.release_channel = self.get_arg("release_channel")
self.release_channel = options["test_parameters"]["release_channel"]
self.architecture = options["test_parameters"]["architecture"]
self.single_date = options["test_parameters"]["single_date"]
self.date_range = options["test_parameters"]["date_range"]
self.startup_cache = options["test_parameters"]["startup_cache"]
@ -206,28 +195,19 @@ class AndroidStartUp(AndroidDevice):
self.proc_start = re.compile(
rf"ActivityManager: Start proc \d+:{self.package_id}/"
)
self.key_name = f"{self.product}_nightly_{self.architecture}.apk"
apk_metadata = self.apk_metadata
self.get_measurements(apk_metadata, metadata)
self.get_measurements(metadata)
# Cleanup
self.device.shell(f"rm {apk_metadata[KEY_NAME]}")
self.device.shell(f"rm {self.key_name}")
return metadata
def get_measurements(self, apk_metadata, metadata):
measurements = self.run_performance_analysis(apk_metadata)
def get_measurements(self, metadata):
measurements = self.install_apk_onto_device_and_run()
self.add_to_metadata(measurements, metadata)
def get_date_array_for_range(self, start, end):
startdate = datetime.strptime(start, DATETIME_FORMAT)
enddate = datetime.strptime(end, DATETIME_FORMAT)
delta_dates = (enddate - startdate).days + 1
return [
(startdate + timedelta(days=i)).strftime("%Y.%m.%d")
for i in range(delta_dates)
]
def add_to_metadata(self, measurements, metadata):
if measurements is not None:
for key, value in measurements.items():
@ -247,14 +227,13 @@ class AndroidStartUp(AndroidDevice):
}
)
def run_performance_analysis(self, apk_metadata):
# Installing the application on the device and getting ready to run the tests
install_path = apk_metadata[KEY_NAME]
self.apk_name = apk_metadata[KEY_NAME].split(".")[0]
if self.custom_apk_exists():
install_path = self.custom_apk_path
if self.product in MOZILLA_PRODUCTS:
def install_apk_onto_device_and_run(self):
if self.product in MOZILLA_PRODUCTS and ON_TRY:
# Installing the application on the device and getting ready to run the tests
if self.custom_apk_exists():
install_path = self.custom_apk_path
else:
install_path = str(self.get_install_path())
self.device.uninstall_app(self.package_id)
self.info(f"Installing {install_path}...")
app_name = self.device.install_app(install_path)
@ -262,14 +241,27 @@ class AndroidStartUp(AndroidDevice):
self.info(f"Successfully installed {app_name}")
else:
raise AndroidStartUpInstallError("The android app was not installed")
if not self.device.is_app_installed(self.package_id):
raise AndroidStartUpInstallError("Please verify your app is installed")
return self.run_tests()
def get_install_path(self):
prefix = pathlib.Path(os.getenv("MOZ_FETCHES_DIR", ""))
if self.product == "fenix":
return prefix / "target.arm64-v8a.apk"
elif self.product == "geckoview_example":
return prefix / "geckoview_example.apk"
elif self.product == "focus":
return prefix / "target.arm64-v8a.apk"
else:
raise AndroidStartUpInstallError("Unknown product")
def run_tests(self):
measurements = {}
# Iterate through the tests in the test list
self.info(f"Running {self.test_name} on {self.apk_name}...")
self.skip_onboarding(self.test_name)
self.info(f"Running {self.test_name} on {self.package_id}...")
time.sleep(self.get_warmup_delay_seconds())
self.skip_onboarding(self.test_name)
test_measurements = []
for i in range(self.test_cycles):
@ -282,6 +274,7 @@ class AndroidStartUp(AndroidDevice):
process = self.device.shell_output(start_cmd_args).splitlines()
test_measurements.append(self.get_measurement(self.test_name, process))
self.device.stop_application(self.package_id)
self.info(f"{self.test_name}: {str(test_measurements)}")
measurements[f"{self.test_name}.{MEASUREMENT_DATA[0]}"] = statistics.mean(
test_measurements
@ -417,6 +410,8 @@ class AndroidStartUp(AndroidDevice):
result_output = self.device.shell_output(resolve_component_args)
stdout = result_output.splitlines()
if len(stdout) != STDOUT_LINE_COUNT: # Should be 2
if "No activity found" in stdout:
raise AndroidStartUpInstallError("Please verify your apk is installed")
raise AndroidStartUpMatchingError(f"expected 2 lines. Got: {stdout}")
return stdout[1]
@ -425,13 +420,22 @@ class AndroidStartUp(AndroidDevice):
We skip onboarding for focus in measure_start_up.py because it's stateful
and needs to be called for every cold start intent.
Onboarding only visibly gets in the way of our MAIN test results.
Additionally, the code block with pm grant enables notifications for the app,
otherwise during testing a request to enable/disable notifications will persist
through app shutdowns.
"""
self.device.shell(
f"pm grant {self.package_id} android.permission.POST_NOTIFICATIONS"
)
if self.product == PROD_FOCUS or test_name not in {
TEST_COLD_MAIN_FF,
TEST_COLD_MAIN_RESTORE,
}:
return
if self.product == MOZILLA_PRODUCTS:
if self.product in MOZILLA_PRODUCTS:
# This sets mutable state so we only need to pass this flag once, before we start the test
self.device.shell(
f"am start-activity -W -a android.intent.action.MAIN --ez "

4
python/mozperftest/mozperftest/tests/data/samples/test_perftest_android_startup.js
Просмотреть файл

@ -28,7 +28,9 @@ module.exports = {
single_date: null, // Dates in YYYY.MM.DD format
date_range: [], // 2 Dates in YYYY.MM.DD format the first and last date(inclusive)
startup_cache: true,
test_cycles: 50,
test_cycles: 5,
release_channel: "nightly", // either release, nightly, beta, or debug
architecture: "arm64-v8a",
},
},
};

171
python/mozperftest/mozperftest/tests/test_android_startup.py
Просмотреть файл

@ -1,14 +1,10 @@
import copy
import json
import pathlib
import random
import time
from datetime import date
from unittest import mock
import mozunit
import pytest
import requests
from mozperftest.system import android_startup
from mozperftest.system.android_startup import (
@ -22,20 +18,11 @@ from mozperftest.tests.support import (
temp_file,
)
SAMPLE_APK_METADATA = {
"name": "fenix_nightly_armeabi-v7a_2022_09_27.apk",
"date": date(2022, 9, 27),
"commit": "",
"architecture": "armeabi-v7a",
"product": "fenix",
}
catch_command = False
ARGS = {
"AndroidStartUp-test-name": "cold_view_nav_start",
"AndroidStartUp-test-name": "cold_main_first_frame",
"AndroidStartUp-product": "fenix",
"AndroidStartUp-release-channel": "nightly",
"apk_metadata": SAMPLE_APK_METADATA,
"test-date": "2023.01.01",
}
@ -52,7 +39,7 @@ class FakeDevice:
def is_app_installed(self, name):
self.name = name
if name == "is_app_installed_fail":
if "not_installed_name" == name or "geckoview_example.apk" in name:
return False
else:
return True
@ -64,6 +51,8 @@ class FakeDevice:
pass
def shell_output(self, cmd):
if self.name == "bad_name":
return ["No activity found"]
if cmd == "logcat -d":
return (
"ActivityManager: Start proc 23943:org.mozilla.fenix/u0a283 \n"
@ -73,55 +62,71 @@ class FakeDevice:
"11-23 14:10:13.391 13135 13135 I "
"GeckoSession: handleMessage GeckoView:PageStart uri="
)
if self.name == "name_for_intent_not_2_lines":
if (
cmd
== "cmd package resolve-activity --brief -a android.intent.action.VIEW -d https://example.com org.mozilla.focus"
and "focus" in self.name
):
return "3 \n lines \n not 2"
elif self.name == "name_for_multiple_Totaltime_strings":
return "2 lines but \n no TotalTime"
elif self.name == "name_for_single_total_time":
return "TotalTime: 123 \n test"
def setup_metadata(metadata, **kwargs):
new_metadata = copy.copy(metadata)
for key, value in kwargs.items():
new_metadata[key] = value
return new_metadata
else:
return "2 lines that \n is all"
def running_env(**kw):
return get_running_env(flavor="mobile-browser", **kw)
def init_mocked_request(status_code, **kwargs):
mock_data = {}
for key, value in kwargs.items():
mock_data["data"][key] = value
mock_request = requests.Response()
mock_request.status_code = status_code
mock_request._content = json.dumps(mock_data).encode("utf-8")
return mock_request
@mock.patch(
"mozperftest.system.android.AndroidDevice.custom_apk_exists", new=lambda x: False
"mozperftest.system.android_startup.PROD_TO_CHANNEL_TO_PKGID",
{"app_name": {"nightly": "not_installed_name"}},
)
@mock.patch(
"mozperftest.system.android_startup.MOZILLA_PRODUCTS",
["app_name"],
)
@mock.patch(
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("requests.get", return_value=init_mocked_request(200))
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_install_of_nightly_failed(*mocked):
SAMPLE_APK_METADATA["name"] = "is_app_installed_fail"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
def test_install_failed(*mocked):
ARGS["AndroidStartUp-product"] = "app_name"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpInstallError):
test.run(metadata)
pass
@mock.patch(
"mozperftest.system.android_startup.PROD_TO_CHANNEL_TO_PKGID",
{"app_name": {"nightly": "not_installed_name"}},
)
@mock.patch(
"mozperftest.system.android_startup.MOZILLA_PRODUCTS",
["app_name"],
)
@mock.patch(
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", True)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_install_failed_CI(*mocked):
ARGS["AndroidStartUp-product"] = "app_name"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpInstallError):
test.run(metadata)
SAMPLE_APK_METADATA["name"] = "fenix_nightly_armeabi-v7a_2022_09_27.apk"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
pass
@ -129,16 +134,63 @@ def test_install_of_nightly_failed(*mocked):
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", True)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_app_not_found_CI(*mocked):
ARGS["AndroidStartUp-product"] = "geckoview_example"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpInstallError):
test.run(metadata)
pass
@mock.patch(
"mozperftest.system.android_startup.PROD_TO_CHANNEL_TO_PKGID",
{"app_name": {"nightly": "not_installed_name"}},
)
@mock.patch(
"mozperftest.system.android_startup.MOZILLA_PRODUCTS",
["app_name"],
)
@mock.patch(
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.utils.ON_TRY", False)
@mock.patch(
"mozperftest.system.android_startup.AndroidStartUp.get_install_path",
return_value="not_installed_name",
)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_install_failed_local(*mocked):
ARGS["AndroidStartUp-product"] = "app_name"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpInstallError):
test.run(metadata)
pass
@mock.patch(
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_invalid_test_name(*mocked):
ARGS["AndroidStartUp-test-name"] = "BAD TEST NAME"
ARGS["AndroidStartUp-product"] = "fenix"
ARGS["AndroidStartUp-test-name"] = "Invalid Test name"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpUnknownTestError):
test.run(metadata)
ARGS["AndroidStartUp-test-name"] = "cold_main_first_frame"
pass
@ -149,18 +201,17 @@ def test_invalid_test_name(*mocked):
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_multiple_matching_lines(*mocked):
SAMPLE_APK_METADATA["name"] = "name_for_intent_not_2_lines"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
ARGS["AndroidStartUp-product"] = "focus"
ARGS["AndroidStartUp-test-name"] = "cold_view_nav_start"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpMatchingError):
test.run(metadata)
SAMPLE_APK_METADATA["name"] = "fenix_nightly_armeabi-v7a_2022_09_27.apk"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
pass
@ -171,18 +222,17 @@ def test_multiple_matching_lines(*mocked):
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
@mock.patch("time.sleep", return_value=time.sleep(0))
@mock.patch("mozperftest.utils.ON_TRY", True)
def test_multiple_total_time_prefix(*mocked):
SAMPLE_APK_METADATA["name"] = "name_for_multiple_Totaltime_strings"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
ARGS["AndroidStartUp-test-name"] = "cold_main_first_frame"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpMatchingError):
test.run(metadata)
SAMPLE_APK_METADATA["name"] = "fenix_nightly_armeabi-v7a_2022_09_27.apk"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
pass
@ -193,19 +243,15 @@ def test_multiple_total_time_prefix(*mocked):
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
@mock.patch("time.sleep", return_value=time.sleep(0))
def test_multiple_start_proc_lines(*mocked):
SAMPLE_APK_METADATA["name"] = "name_for_multiple_Totaltime_strings"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
test = android_startup.AndroidStartUp(env, mach_cmd)
with pytest.raises(AndroidStartUpMatchingError):
test.run(metadata)
SAMPLE_APK_METADATA["name"] = "fenix_nightly_armeabi-v7a_2022_09_27.apk"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
pass
@ -216,14 +262,14 @@ def test_multiple_start_proc_lines(*mocked):
"mozdevice.ADBDevice",
new=FakeDevice,
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
@mock.patch("time.sleep", return_value=time.sleep(0))
@mock.patch(
"mozperftest.system.android_startup.AndroidStartUp.get_measurement",
return_value=random.randint(500, 1000),
)
def test_perfherder_layer(*mocked):
SAMPLE_APK_METADATA["name"] = "name_for_multiple_Totaltime_strings"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
ARGS["AndroidStartUp-product"] = "fenix"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
@ -241,9 +287,8 @@ def test_perfherder_layer(*mocked):
"mozperftest.system.android_startup.AndroidStartUp.get_measurement",
return_value=random.randint(500, 1000),
)
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
def test_custom_apk_startup(get_measurement_mock, time_sleep_mock, path_mock):
SAMPLE_APK_METADATA["name"] = "name_for_multiple_Totaltime_strings"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS
)
@ -259,7 +304,7 @@ def test_custom_apk_startup(get_measurement_mock, time_sleep_mock, path_mock):
test.run_tests = lambda: True
test.package_id = "FakeID"
test.product = "fenix"
assert test.run_performance_analysis(SAMPLE_APK_METADATA)
assert test.install_apk_onto_device_and_run()
@mock.patch(
@ -270,9 +315,9 @@ def test_custom_apk_startup(get_measurement_mock, time_sleep_mock, path_mock):
new=FakeDevice,
)
@mock.patch("time.sleep", return_value=time.sleep(0))
@mock.patch("mozperftest.system.android_startup.ON_TRY", False)
def test_get_measurement_from_nav_start_logcat_match_error(*mocked):
SAMPLE_APK_METADATA["name"] = "name_for_single_total_time"
ARGS["apk_metadata"] = SAMPLE_APK_METADATA
ARGS["AndroidStartUp-product"] = "focus"
ARGS["AndroidStartUp-test-name"] = "cold_view_nav_start"
mach_cmd, metadata, env = running_env(
tests=[str(EXAMPLE_ANDROID_STARTUP_TEST)], **ARGS

61
taskcluster/kinds/perftest/android.yml
Просмотреть файл

@ -48,6 +48,12 @@ hw-a51-startup-fenix-cold-main-first-frame:
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
fetches:
build:
- artifact: target.arm64-v8a.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: signing-apk-fenix-nightly-simulation
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -59,10 +65,7 @@ hw-a51-startup-fenix-cold-main-first-frame:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_main_first_frame
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=fenix
--AndroidStartUp-release-channel=nightly
hw-a51-startup-fenix-cold-view-nav-start:
worker-type: t-bitbar-gw-perf-a51
@ -73,6 +76,12 @@ hw-a51-startup-fenix-cold-view-nav-start:
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
fetches:
build:
- artifact: target.arm64-v8a.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: signing-apk-fenix-nightly-simulation
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -84,10 +93,7 @@ hw-a51-startup-fenix-cold-view-nav-start:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_view_nav_start
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=fenix
--AndroidStartUp-release-channel=nightly
hw-a51-startup-focus-cold-main-first-frame:
worker-type: t-bitbar-gw-perf-a51
@ -97,7 +103,13 @@ hw-a51-startup-focus-cold-main-first-frame:
tier: 2
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
cron: false
fetches:
build:
- artifact: target.arm64-v8a.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: signing-apk-focus-nightly
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -109,10 +121,7 @@ hw-a51-startup-focus-cold-main-first-frame:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_main_first_frame
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=focus
--AndroidStartUp-release-channel=nightly
hw-a51-startup-focus-cold-view-nav-start:
worker-type: t-bitbar-gw-perf-a51
@ -122,7 +131,13 @@ hw-a51-startup-focus-cold-view-nav-start:
tier: 2
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
cron: false
fetches:
build:
- artifact: target.arm64-v8a.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: signing-apk-focus-nightly
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -134,10 +149,7 @@ hw-a51-startup-focus-cold-view-nav-start:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_view_nav_start
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=focus
--AndroidStartUp-release-channel=nightly
hw-a51-startup-geckoview-cold-main-first-frame:
worker-type: t-bitbar-gw-perf-a51
@ -148,6 +160,12 @@ hw-a51-startup-geckoview-cold-main-first-frame:
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
fetches:
build:
- artifact: geckoview_example.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: build-android-aarch64-shippable/opt
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -159,10 +177,7 @@ hw-a51-startup-geckoview-cold-main-first-frame:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_main_first_frame
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=geckoview_example
--AndroidStartUp-release-channel=nightly
hw-a51-startup-geckoview-cold-view-nav-start:
worker-type: t-bitbar-gw-perf-a51
@ -173,6 +188,12 @@ hw-a51-startup-geckoview-cold-view-nav-start:
platform: android-hw-a51-11-0-aarch64-shippable/opt
attributes:
cron: true
fetches:
build:
- artifact: geckoview_example.apk
extract: false
require-build:
android-hw-a51-11-0-aarch64-shippable/opt: build-android-aarch64-shippable/opt
run:
command: >-
mkdir -p $MOZ_FETCHES_DIR/../artifacts &&
@ -184,10 +205,7 @@ hw-a51-startup-geckoview-cold-view-nav-start:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_view_nav_start
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=geckoview_example
--AndroidStartUp-release-channel=nightly
hw-a51-startup-chrome-m-cold-main-first-frame:
worker-type: t-bitbar-gw-perf-a51
@ -209,10 +227,7 @@ hw-a51-startup-chrome-m-cold-main-first-frame:
--browsertime-cycles=0
--AndroidStartUp-test-name=cold_main_first_frame
--perfherder
--hooks
testing/performance/hooks_android_startup.py
--AndroidStartUp-product=chrome-m
--AndroidStartUp-release-channel=release
hw-a51-perfstats-gv:
worker-type: t-bitbar-gw-perf-a51

2
taskcluster/kinds/perftest/kind.yml
Просмотреть файл

@ -7,6 +7,8 @@ loader: gecko_taskgraph.loader.transform:loader
kind-dependencies:
- toolchain
- build
- build-apk
- signing-apk
transforms:
- gecko_taskgraph.transforms.perftest:transforms

71
testing/performance/hooks_android_startup.py
Просмотреть файл

@ -1,71 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import pathlib
import re
import subprocess
from datetime import datetime, timedelta
import requests
from mozperftest.system.android_startup import (
BASE_URL_DICT,
DATETIME_FORMAT,
KEY_ARCHITECTURE,
KEY_COMMIT,
KEY_DATETIME,
KEY_NAME,
KEY_PRODUCT,
MOZILLA_PRODUCTS,
)
from mozversioncontrol import get_repository_object
HTTP_200_OKAY = 200
MONO_REPO_MIGRATION_DAY = datetime(2024, 3, 18)
def before_iterations(kw):
product = kw["AndroidStartUp_product"]
architecture = "arm64-v8a"
if product == "geckoview_example":
architecture = "aarch64"
if get_repository_object("").name == "git":
commit_info = subprocess.getoutput("git log --max-count 1")
else:
commit_info = subprocess.getoutput("hg log -l 1")
commit_date = re.search(r"[Dd]ate:\s+([:\s\w]+)\s+", str(commit_info)).group(1)
download_date = datetime.strptime(commit_date, "%a %b %d %H:%M:%S %Y") - timedelta(
days=1
)
pre_mono_repo = ""
if download_date < MONO_REPO_MIGRATION_DAY and product != "geckoview_example":
pre_mono_repo = "-pre-mono-repo"
download_date = download_date.strftime(DATETIME_FORMAT)
if product in MOZILLA_PRODUCTS:
nightly_url = BASE_URL_DICT[product + pre_mono_repo].format(
date=download_date, architecture=architecture
)
filename = f"{product}_nightly_{architecture}.apk"
print("Fetching {}...".format(filename), end="", flush=True)
download_apk_as_date(nightly_url, download_date, filename)
print(f"Downloaded {product} for date: {download_date}")
else:
filename = product + ".apk"
kw["apk_metadata"] = {
KEY_NAME: filename,
KEY_DATETIME: download_date,
KEY_COMMIT: "",
KEY_ARCHITECTURE: architecture,
KEY_PRODUCT: product,
}
def download_apk_as_date(nightly_url, download_date_string, filename):
apk = requests.get(nightly_url)
if apk.status_code != HTTP_200_OKAY:
raise Exception(
f"Something went wrong downloading the apk check to make sure you have entered"
f" a date that is valid and that the apk for the date you have requested("
f"{download_date_string}) is available and that the URL({nightly_url}) is also "
f"valid"
)
pathlib.Path(filename).write_bytes(apk.content)

2
testing/performance/perftest_android_startup.js
Просмотреть файл

@ -28,6 +28,8 @@ module.exports = {
date_range: [], // 2 Dates in YYYY.MM.DD format the first and last date(inclusive)
startup_cache: true,
test_cycles: 50,
release_channel: "nightly", // either release, nightly, beta, or debug
architecture: "arm64-v8a",
},
},
};