зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1786092 - Resolve a few miscalleanous WPT bugs. r=sparky,perftest-reviewers
What are we doing: - Resolving a few bugs/user requests Issues being addressed: - Resolved issue where if the WPT_key.txt file is not available locally it does not affect running ./mach perftest-test - Added section to WPT where we display the amount of tests we have remaining - Altered the request_with_timeout function, to better handle requests Differential Revision: https://phabricator.services.mozilla.com/D155268
This commit is contained in:
Родитель
d25798f6e6
Коммит
2afa39dd24
|
@ -28,6 +28,7 @@ ACCEPTED_CONNECTIONS = [
|
||||||
]
|
]
|
||||||
|
|
||||||
ACCEPTED_STATISTICS = ["average", "median", "standardDeviation"]
|
ACCEPTED_STATISTICS = ["average", "median", "standardDeviation"]
|
||||||
|
WPT_KEY_FILE = "WPT_key.txt"
|
||||||
|
|
||||||
|
|
||||||
class WPTTimeOutError(Exception):
|
class WPTTimeOutError(Exception):
|
||||||
|
@ -169,10 +170,10 @@ class WebPageTest(Layer):
|
||||||
if utils.ON_TRY:
|
if utils.ON_TRY:
|
||||||
self.WPT_key = utils.get_tc_secret(wpt=True)["wpt_key"]
|
self.WPT_key = utils.get_tc_secret(wpt=True)["wpt_key"]
|
||||||
else:
|
else:
|
||||||
self.WPT_key = pathlib.Path(HERE, "WPT_key.txt").open().read()
|
self.WPT_key = pathlib.Path(HERE, WPT_KEY_FILE).open().read()
|
||||||
self.statistic_types = ["average", "median", "standardDeviation"]
|
self.statistic_types = ["average", "median", "standardDeviation"]
|
||||||
self.timeout_limit = 21600
|
self.timeout_limit = 21600
|
||||||
self.wait_between_requests = 5
|
self.wait_between_requests = 180
|
||||||
|
|
||||||
def run(self, metadata):
|
def run(self, metadata):
|
||||||
options = metadata.script["options"]
|
options = metadata.script["options"]
|
||||||
|
@ -181,6 +182,7 @@ class WebPageTest(Layer):
|
||||||
self.wpt_browser_metrics = options["browser_metrics"]
|
self.wpt_browser_metrics = options["browser_metrics"]
|
||||||
self.pre_run_error_checks(options["test_parameters"], test_list)
|
self.pre_run_error_checks(options["test_parameters"], test_list)
|
||||||
self.create_and_run_wpt_threaded_tests(test_list, metadata)
|
self.create_and_run_wpt_threaded_tests(test_list, metadata)
|
||||||
|
self.test_runs_left_this_month()
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
def pre_run_error_checks(self, options, test_list):
|
def pre_run_error_checks(self, options, test_list):
|
||||||
|
@ -239,19 +241,25 @@ class WebPageTest(Layer):
|
||||||
)
|
)
|
||||||
|
|
||||||
def request_with_timeout(self, url):
|
def request_with_timeout(self, url):
|
||||||
results_of_test = {}
|
requested_results = requests.get(url)
|
||||||
|
results_of_request = json.loads(requested_results.text)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
while (
|
while (
|
||||||
results_of_test.get("statusCode") != 200
|
requested_results.status_code == 200
|
||||||
and time.time() - start < self.timeout_limit
|
and time.time() - start < self.timeout_limit
|
||||||
|
and (
|
||||||
|
"statusCode" in results_of_request.keys()
|
||||||
|
and results_of_request["statusCode"] != 200
|
||||||
|
)
|
||||||
):
|
):
|
||||||
results_of_test = json.loads(requests.get(url).text)
|
requested_results = requests.get(url)
|
||||||
|
results_of_request = json.loads(requested_results.text)
|
||||||
time.sleep(self.wait_between_requests)
|
time.sleep(self.wait_between_requests)
|
||||||
if results_of_test.get("statusCode") != 200:
|
if time.time() - start > self.timeout_limit:
|
||||||
raise WPTTimeOutError(
|
raise WPTTimeOutError(
|
||||||
f"{url} test timed out after {self.timeout_limit} seconds"
|
f"{url} test timed out after {self.timeout_limit} seconds"
|
||||||
)
|
)
|
||||||
return results_of_test
|
return results_of_request
|
||||||
|
|
||||||
def check_urls_are_valid(self, test_list):
|
def check_urls_are_valid(self, test_list):
|
||||||
for url in test_list:
|
for url in test_list:
|
||||||
|
@ -376,3 +384,11 @@ class WebPageTest(Layer):
|
||||||
except Exception:
|
except Exception:
|
||||||
self.error("Issue found with processing browser/WPT version")
|
self.error("Issue found with processing browser/WPT version")
|
||||||
return desired_values
|
return desired_values
|
||||||
|
|
||||||
|
def test_runs_left_this_month(self):
|
||||||
|
tests_left_this_month = self.request_with_timeout(
|
||||||
|
f"https://www.webpagetest.org/testBalance.php?k={self.WPT_key}&f=json"
|
||||||
|
)
|
||||||
|
self.info(
|
||||||
|
f"There are {tests_left_this_month['data']['remaining']} tests remaining"
|
||||||
|
)
|
||||||
|
|
|
@ -82,19 +82,21 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False):
|
||||||
return placeholder_data
|
return placeholder_data
|
||||||
|
|
||||||
|
|
||||||
def init_mocked_request(status_code, **kwargs):
|
def init_mocked_request(status_code, WPT_test_status_code=200, **kwargs):
|
||||||
mock_data = {
|
mock_data = {
|
||||||
"data": {
|
"data": {
|
||||||
"ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
|
"ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"},
|
||||||
"jsonUrl": "mock_test.com",
|
"jsonUrl": "mock_test.com",
|
||||||
"summary": "Just a pageload test",
|
"summary": "Just a pageload test",
|
||||||
"url": "testurl.ca",
|
"url": "testurl.ca",
|
||||||
|
"remaining": 2000,
|
||||||
},
|
},
|
||||||
"statusCode": status_code,
|
"statusCode": WPT_test_status_code,
|
||||||
}
|
}
|
||||||
for key, value in kwargs.items():
|
for key, value in kwargs.items():
|
||||||
mock_data["data"][key] = value
|
mock_data["data"][key] = value
|
||||||
mock_request = requests.Response()
|
mock_request = requests.Response()
|
||||||
|
mock_request.status_code = status_code
|
||||||
mock_request._content = json.dumps(mock_data).encode("utf-8")
|
mock_request._content = json.dumps(mock_data).encode("utf-8")
|
||||||
return mock_request
|
return mock_request
|
||||||
|
|
||||||
|
@ -107,6 +109,8 @@ def init_mocked_request(status_code, **kwargs):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_no_issues_mocked_results(*mocked):
|
def test_webpagetest_no_issues_mocked_results(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -120,6 +124,8 @@ def test_webpagetest_no_issues_mocked_results(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_invalid_browser(*mocked):
|
def test_webpagetest_test_invalid_browser(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser"
|
metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser"
|
||||||
|
@ -132,6 +138,8 @@ def test_webpagetest_test_invalid_browser(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_invalid_connection(*mocked):
|
def test_webpagetest_test_invalid_connection(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -144,6 +152,8 @@ def test_webpagetest_test_invalid_connection(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_invalid_url(*mocked):
|
def test_webpagetest_test_invalid_url(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -156,6 +166,8 @@ def test_webpagetest_test_invalid_url(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_invalid_statistic(*mocked):
|
def test_webpagetest_test_invalid_statistic(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -171,6 +183,7 @@ def test_webpagetest_test_invalid_statistic(*mocked):
|
||||||
"mozperftest.test.webpagetest.WebPageTest.request_with_timeout",
|
"mozperftest.test.webpagetest.WebPageTest.request_with_timeout",
|
||||||
return_value={"data": {}},
|
return_value={"data": {}},
|
||||||
)
|
)
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_invalid_location(*mocked):
|
def test_webpagetest_test_invalid_location(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -179,11 +192,12 @@ def test_webpagetest_test_invalid_location(*mocked):
|
||||||
test.run(metadata)
|
test.run(metadata)
|
||||||
|
|
||||||
|
|
||||||
@mock.patch("requests.get", return_value=init_mocked_request(400))
|
@mock.patch("requests.get", return_value=init_mocked_request(200, 101))
|
||||||
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
@mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"})
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_timeout(*mocked):
|
def test_webpagetest_test_timeout(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
test = webpagetest.WebPageTest(env, mach_cmd)
|
test = webpagetest.WebPageTest(env, mach_cmd)
|
||||||
|
@ -204,6 +218,7 @@ def test_webpagetest_test_timeout(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_wrong_browserlocation(*mocked):
|
def test_webpagetest_test_wrong_browserlocation(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||||
|
@ -221,6 +236,8 @@ def test_webpagetest_test_wrong_browserlocation(*mocked):
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
"mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None
|
||||||
)
|
)
|
||||||
|
@mock.patch("requests.get", return_value=init_mocked_request(200))
|
||||||
|
@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt")
|
||||||
def test_webpagetest_test_metric_not_found(*mocked):
|
def test_webpagetest_test_metric_not_found(*mocked):
|
||||||
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)])
|
||||||
metadata.script["options"]["test_list"] = ["google.ca"]
|
metadata.script["options"]["test_list"] = ["google.ca"]
|
||||||
|
|
Загрузка…
Ссылка в новой задаче