diff --git a/python/mozperftest/mozperftest/test/webpagetest.py b/python/mozperftest/mozperftest/test/webpagetest.py index a2b615b3f1a9..efd45c1a56f7 100644 --- a/python/mozperftest/mozperftest/test/webpagetest.py +++ b/python/mozperftest/mozperftest/test/webpagetest.py @@ -28,6 +28,7 @@ ACCEPTED_CONNECTIONS = [ ] ACCEPTED_STATISTICS = ["average", "median", "standardDeviation"] +WPT_KEY_FILE = "WPT_key.txt" class WPTTimeOutError(Exception): @@ -169,10 +170,10 @@ class WebPageTest(Layer): if utils.ON_TRY: self.WPT_key = utils.get_tc_secret(wpt=True)["wpt_key"] else: - self.WPT_key = pathlib.Path(HERE, "WPT_key.txt").open().read() + self.WPT_key = pathlib.Path(HERE, WPT_KEY_FILE).open().read() self.statistic_types = ["average", "median", "standardDeviation"] self.timeout_limit = 21600 - self.wait_between_requests = 5 + self.wait_between_requests = 180 def run(self, metadata): options = metadata.script["options"] @@ -181,6 +182,7 @@ class WebPageTest(Layer): self.wpt_browser_metrics = options["browser_metrics"] self.pre_run_error_checks(options["test_parameters"], test_list) self.create_and_run_wpt_threaded_tests(test_list, metadata) + self.test_runs_left_this_month() return metadata def pre_run_error_checks(self, options, test_list): @@ -239,19 +241,25 @@ class WebPageTest(Layer): ) def request_with_timeout(self, url): - results_of_test = {} + requested_results = requests.get(url) + results_of_request = json.loads(requested_results.text) start = time.time() while ( - results_of_test.get("statusCode") != 200 + requested_results.status_code == 200 and time.time() - start < self.timeout_limit + and ( + "statusCode" in results_of_request.keys() + and results_of_request["statusCode"] != 200 + ) ): - results_of_test = json.loads(requests.get(url).text) + requested_results = requests.get(url) + results_of_request = json.loads(requested_results.text) time.sleep(self.wait_between_requests) - if results_of_test.get("statusCode") != 200: + if time.time() - start > self.timeout_limit: raise WPTTimeOutError( f"{url} test timed out after {self.timeout_limit} seconds" ) - return results_of_test + return results_of_request def check_urls_are_valid(self, test_list): for url in test_list: @@ -376,3 +384,11 @@ class WebPageTest(Layer): except Exception: self.error("Issue found with processing browser/WPT version") return desired_values + + def test_runs_left_this_month(self): + tests_left_this_month = self.request_with_timeout( + f"https://www.webpagetest.org/testBalance.php?k={self.WPT_key}&f=json" + ) + self.info( + f"There are {tests_left_this_month['data']['remaining']} tests remaining" + ) diff --git a/python/mozperftest/mozperftest/tests/data/WPT_fakekey.txt b/python/mozperftest/mozperftest/tests/data/WPT_fakekey.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/mozperftest/mozperftest/tests/test_webpagetest.py b/python/mozperftest/mozperftest/tests/test_webpagetest.py index d8fd51c429b2..4628f99cfe0d 100644 --- a/python/mozperftest/mozperftest/tests/test_webpagetest.py +++ b/python/mozperftest/mozperftest/tests/test_webpagetest.py @@ -82,19 +82,21 @@ def init_placeholder_wpt_data(fvonly=False, invalid_results=False): return placeholder_data -def init_mocked_request(status_code, **kwargs): +def init_mocked_request(status_code, WPT_test_status_code=200, **kwargs): mock_data = { "data": { "ec2-us-east-1": {"PendingTests": {"Queued": 3}, "Label": "California"}, "jsonUrl": "mock_test.com", "summary": "Just a pageload test", "url": "testurl.ca", + "remaining": 2000, }, - "statusCode": status_code, + "statusCode": WPT_test_status_code, } for key, value in kwargs.items(): mock_data["data"][key] = value mock_request = requests.Response() + mock_request.status_code = status_code mock_request._content = json.dumps(mock_data).encode("utf-8") return mock_request @@ -107,6 +109,8 @@ def init_mocked_request(status_code, **kwargs): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_no_issues_mocked_results(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -120,6 +124,8 @@ def test_webpagetest_no_issues_mocked_results(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_invalid_browser(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) metadata.script["options"]["test_parameters"]["browser"] = "Invalid Browser" @@ -132,6 +138,8 @@ def test_webpagetest_test_invalid_browser(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_invalid_connection(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -144,6 +152,8 @@ def test_webpagetest_test_invalid_connection(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_invalid_url(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -156,6 +166,8 @@ def test_webpagetest_test_invalid_url(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_invalid_statistic(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -171,6 +183,7 @@ def test_webpagetest_test_invalid_statistic(*mocked): "mozperftest.test.webpagetest.WebPageTest.request_with_timeout", return_value={"data": {}}, ) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_invalid_location(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -179,11 +192,12 @@ def test_webpagetest_test_invalid_location(*mocked): test.run(metadata) -@mock.patch("requests.get", return_value=init_mocked_request(400)) +@mock.patch("requests.get", return_value=init_mocked_request(200, 101)) @mock.patch("mozperftest.utils.get_tc_secret", return_value={"wpt_key": "fake_key"}) @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_timeout(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) test = webpagetest.WebPageTest(env, mach_cmd) @@ -204,6 +218,7 @@ def test_webpagetest_test_timeout(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_wrong_browserlocation(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) metadata.script["options"]["test_list"] = ["google.ca"] @@ -221,6 +236,8 @@ def test_webpagetest_test_wrong_browserlocation(*mocked): @mock.patch( "mozperftest.test.webpagetest.WebPageTest.location_queue", return_value=None ) +@mock.patch("requests.get", return_value=init_mocked_request(200)) +@mock.patch("mozperftest.test.webpagetest.WPT_KEY_FILE", "tests/data/WPT_fakekey.txt") def test_webpagetest_test_metric_not_found(*mocked): mach_cmd, metadata, env = running_env(tests=[str(EXAMPLE_WPT_TEST)]) metadata.script["options"]["test_list"] = ["google.ca"]