From 1be090fa36b61005dffe616e6d5361849626dd2a Mon Sep 17 00:00:00 2001 From: Csoregi Natalia Date: Thu, 30 Sep 2021 21:33:34 +0300 Subject: [PATCH] Backed out 3 changesets (bug 1713815) for causing perfdoc failures. CLOSED TREE Backed out changeset f57299491e76 (bug 1713815) Backed out changeset a6bbbb4b43f1 (bug 1713815) Backed out changeset 5672d15a0b2c (bug 1713815) --- taskcluster/ci/test/browsertime-desktop.yml | 47 ------ taskcluster/ci/test/test-sets.yml | 1 - .../visual-metrics/run-visual-metrics.py | 5 +- .../manifestparser/manifestparser/util.py | 21 --- .../mozbase/manifestparser/tests/manifest.ini | 1 - .../mozbase/manifestparser/tests/test_util.py | 107 ------------- testing/perfdocs/generated/raptor.rst | 145 ------------------ testing/performance/hooks_recording.py | 7 - testing/performance/pageload_sites.json | 74 --------- testing/performance/perftest_record.js | 79 +--------- .../browsertime/browsertime_interactive.js | 104 ------------- .../browsertime/browsertime_pageload.js | 3 +- testing/raptor/raptor/browsertime/base.py | 40 ++--- testing/raptor/raptor/manifest.py | 21 +-- testing/raptor/raptor/output.py | 16 +- testing/raptor/raptor/perfdocs/config.yml | 7 - testing/raptor/raptor/perftest.py | 1 - testing/raptor/raptor/raptor.ini | 3 - testing/raptor/raptor/results.py | 24 +-- .../browsertime-responsiveness.ini | 101 ------------ .../mitm6-windows-firefox-cnn-nav.manifest | 9 -- ...itm6-windows-firefox-facebook-nav.manifest | 9 -- ...dows-firefox-reddit-billgates-ama.manifest | 9 -- ...ows-firefox-reddit-billgates-post.manifest | 9 -- 24 files changed, 28 insertions(+), 815 deletions(-) delete mode 100644 testing/mozbase/manifestparser/tests/test_util.py delete mode 100644 testing/raptor/browsertime/browsertime_interactive.js delete mode 100644 testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini delete mode 100644 testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest delete mode 100644 testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest delete mode 100644 testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest delete mode 100644 testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest diff --git a/taskcluster/ci/test/browsertime-desktop.yml b/taskcluster/ci/test/browsertime-desktop.yml index d29181acc32b..4454cc1176bc 100644 --- a/taskcluster/ci/test/browsertime-desktop.yml +++ b/taskcluster/ci/test/browsertime-desktop.yml @@ -122,53 +122,6 @@ browsertime-tp6: firefox: 2 default: 3 -browsertime-responsiveness: - <<: *tp6-defaults - raptor-subtests: - - cnn-nav - mozharness: - extra-options: - by-test-platform: - windows10-.*: - - --cold - - --browsertime - - --browsertime-no-ffwindowrecorder - - --conditioned-profile=settled - default: - - --cold - - --browsertime - - --conditioned-profile=settled - run-on-projects: - by-variant: - fission: - by-test-platform: - windows.*-32.*: [] - .*clang.*: [] - default: [mozilla-central] - default: - by-app: - firefox: - by-test-platform: - .*clang.*: [] - linux.*shippable.*: [mozilla-central] - macosx1015.*shippable.*: [mozilla-central] - windows10-64.*shippable.*: [mozilla-central] - default: [] - default: [] - variants: - by-app: - firefox: - by-test-platform: - .*clang.*: [] - windows10-64.*shippable-qr/.*: [fission] - (?!windows10-64).*shippable-qr/.*: [fission] - default: [] - default: [] - tier: - by-app: - firefox: 2 - default: 3 - browsertime-tp6-essential: <<: *tp6-defaults raptor-subtests: diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 8569aa642098..c67d41f87699 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -90,7 +90,6 @@ browsertime: - browsertime-youtube-playback - browsertime-custom - browsertime-first-install - - browsertime-responsiveness browsertime-profiling: - browsertime-tp6-profiling diff --git a/taskcluster/docker/visual-metrics/run-visual-metrics.py b/taskcluster/docker/visual-metrics/run-visual-metrics.py index 2aad7fce6ee1..2593a0877ef3 100755 --- a/taskcluster/docker/visual-metrics/run-visual-metrics.py +++ b/taskcluster/docker/visual-metrics/run-visual-metrics.py @@ -320,12 +320,9 @@ def main(log, args): for site in browsertime_json: for video in site["files"]["video"]: count += 1 - name = job["test_name"] - if "alias" in site["info"] and site["info"]["alias"].strip() != "": - name = "%s.%s" % (name, site["info"]["alias"]) jobs.append( Job( - test_name=name, + test_name=job["test_name"], extra_options=len(job["extra_options"]) > 0 and job["extra_options"] or jobs_json["extra_options"], diff --git a/testing/mozbase/manifestparser/manifestparser/util.py b/testing/mozbase/manifestparser/manifestparser/util.py index 36bf33fadf4c..67920cfedf24 100644 --- a/testing/mozbase/manifestparser/manifestparser/util.py +++ b/testing/mozbase/manifestparser/manifestparser/util.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function, unicode_literals -import ast import os @@ -26,23 +25,3 @@ def normsep(path): else: path = path.replace(os.altsep, "/") return path - - -def evaluate_list_from_string(list_string): - """ - This is a utility function for converting a string obtained from a manifest - into a list. If the string is not a valid list when converted, an error will be - raised from `ast.eval_literal`. For example, you can convert entries like this - into a list: - ``` - test_settings= - ["hello", "world"], - [1, 10, 100], - values= - 5, - 6, - 7, - 8, - ``` - """ - return ast.literal_eval("[" + "".join(list_string.strip(",").split("\n")) + "]") diff --git a/testing/mozbase/manifestparser/tests/manifest.ini b/testing/mozbase/manifestparser/tests/manifest.ini index cd8852134c59..432c15c32770 100644 --- a/testing/mozbase/manifestparser/tests/manifest.ini +++ b/testing/mozbase/manifestparser/tests/manifest.ini @@ -10,4 +10,3 @@ subsuite = mozbase [test_convert_symlinks.py] disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938 [test_default_overrides.py] -[test_util.py] diff --git a/testing/mozbase/manifestparser/tests/test_util.py b/testing/mozbase/manifestparser/tests/test_util.py deleted file mode 100644 index 305eec8d8e7d..000000000000 --- a/testing/mozbase/manifestparser/tests/test_util.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python - -""" -Test how our utility functions are working. -""" - -from __future__ import absolute_import - -from textwrap import dedent - -import mozunit -import pytest -from six import StringIO - -from manifestparser import read_ini -from manifestparser.util import evaluate_list_from_string - - -@pytest.fixture(scope="module") -def parse_manifest(): - def inner(string, **kwargs): - buf = StringIO() - buf.write(dedent(string)) - buf.seek(0) - return read_ini(buf, **kwargs)[0] - - return inner - - -@pytest.mark.parametrize( - "test_manifest, expected_list", - [ - [ - """ - [test_felinicity.py] - kittens = true - cats = - "I", - "Am", - "A", - "Cat", - """, - ["I", "Am", "A", "Cat"], - ], - [ - """ - [test_felinicity.py] - kittens = true - cats = - ["I", 1], - ["Am", 2], - ["A", 3], - ["Cat", 4], - """, - [ - ["I", 1], - ["Am", 2], - ["A", 3], - ["Cat", 4], - ], - ], - ], -) -def test_string_to_list_conversion(test_manifest, expected_list, parse_manifest): - parsed_tests = parse_manifest(test_manifest) - assert evaluate_list_from_string(parsed_tests[0][1]["cats"]) == expected_list - - -@pytest.mark.parametrize( - "test_manifest, failure", - [ - [ - """ - # This will fail since the elements are not enlosed in quotes - [test_felinicity.py] - kittens = true - cats = - I, - Am, - A, - Cat, - """, - ValueError, - ], - [ - """ - # This will fail since the syntax is incorrect - [test_felinicity.py] - kittens = true - cats = - ["I", 1, - ["Am", 2, - ["A", 3], - ["Cat", 4], - """, - SyntaxError, - ], - ], -) -def test_string_to_list_conversion_failures(test_manifest, failure, parse_manifest): - parsed_tests = parse_manifest(test_manifest) - with pytest.raises(failure): - evaluate_list_from_string(parsed_tests[0][1]["cats"]) - - -if __name__ == "__main__": - mozunit.main() diff --git a/testing/perfdocs/generated/raptor.rst b/testing/perfdocs/generated/raptor.rst index d8248b3a3ed4..b8f3fc4e4c90 100644 --- a/testing/perfdocs/generated/raptor.rst +++ b/testing/perfdocs/generated/raptor.rst @@ -4217,151 +4217,6 @@ Tests for page-load performance. The links direct to the actual websites that ar -Interactive ------------ -Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest. - -.. dropdown:: cnn-nav (Navigates to cnn main page, then to the world sub-page.) - :container: + anchor-id-cnn-nav-i - - * **accept zero vismet**: true - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **browser cycles**: 10 - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **interactive**: true - * **lower is better**: true - * **measure**: fnbpaint, fcp, dcf, loadtime - * **page cycles**: 25 - * **page timeout**: 60000 - * **playback**: mitmproxy - * **playback pageset manifest**: mitm6-windows-firefox-cnn-nav.manifest - * **playback version**: 6.0.2 - * **test cmds**: ["measure.start", "landing"], ["navigate", "https://www.cnn.com"], ["wait.byTime", 4000], ["measure.stop", ""], ["measure.start", "world"], ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], ["wait.byTime", 1000], ["measure.stop", ""], - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: false - * **Test Task**: - * test-linux1804-64-clang-trunk-qr/opt - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * test-linux1804-64-qr/opt - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * test-linux1804-64-shippable-qr/opt - * browsertime-responsiveness-chrome-cnn-nav-e10s: None - * browsertime-responsiveness-chromium-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central - * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central - * test-macosx1014-64-shippable-qr/opt - * browsertime-responsiveness-chrome-cnn-nav-e10s: None - * browsertime-responsiveness-chromium-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central - * test-macosx1015-64-shippable-qr/opt - * browsertime-responsiveness-chrome-cnn-nav-e10s: None - * browsertime-responsiveness-chromium-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central - * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central - * test-windows10-32-qr/opt - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * test-windows10-32-shippable-qr/opt - * browsertime-responsiveness-chrome-cnn-nav-e10s: None - * browsertime-responsiveness-chromium-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: None - * test-windows10-64-qr/opt - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * test-windows10-64-ref-hw-2017-qr/opt - * browsertime-responsiveness-firefox-cnn-nav-e10s: None - * test-windows10-64-shippable-qr/opt - * browsertime-responsiveness-chrome-cnn-nav-e10s: None - * browsertime-responsiveness-chromium-cnn-nav-e10s: None - * browsertime-responsiveness-firefox-cnn-nav-e10s: mozilla-central - * browsertime-responsiveness-firefox-cnn-nav-fis-e10s: mozilla-central - - -.. dropdown:: facebook-nav (Navigates to facebook, then the sub-pages friends, marketplace, groups.) - :container: + anchor-id-facebook-nav-i - - * **accept zero vismet**: true - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **browser cycles**: 25 - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **interactive**: true - * **lower is better**: true - * **measure**: fnbpaint, fcp, dcf, loadtime - * **page cycles**: 25 - * **page timeout**: 90000 - * **playback**: mitmproxy - * **playback pageset manifest**: mitm6-windows-firefox-facebook-nav.manifest - * **playback version**: 6.0.2 - * **test cmds**: ["measure.start", "landing"], ["navigate", "https://www.facebook.com/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "friends"], ["navigate", "https://www.facebook.com/friends/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "marketplace"], ["navigate", "https://www.facebook.com/marketplace"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "groups"], ["navigate", "https://www.facebook.com/groups/discover/"], ["wait.byTime", 5000], ["measure.stop", ""], - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: false - - -.. dropdown:: reddit-billgates-ama (Navigates from the Bill Gates AMA to the Reddit members section.) - :container: + anchor-id-reddit-billgates-ama-i - - * **accept zero vismet**: true - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **browser cycles**: 25 - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **interactive**: true - * **lower is better**: true - * **measure**: fnbpaint, fcp, dcf, loadtime - * **page cycles**: 25 - * **page timeout**: 60000 - * **playback**: mitmproxy - * **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-ama.manifest - * **playback version**: 6.0.2 - * **test cmds**: ["measure.start", "billg-ama"], ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "members"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], ["wait.byTime", 1000], ["measure.stop", ""], - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: false - - -.. dropdown:: reddit-billgates-post (Navigates the `thisisbillgates` user starting at the main user page, then to the posts, comments, hot, and top sections.) - :container: + anchor-id-reddit-billgates-post-i - - * **accept zero vismet**: true - * **alert on**: fcp, loadtime - * **alert threshold**: 2.0 - * **apps**: firefox, chrome, chromium - * **browser cycles**: 25 - * **expected**: pass - * **gecko profile entries**: 14000000 - * **gecko profile interval**: 1 - * **interactive**: true - * **lower is better**: true - * **measure**: fnbpaint, fcp, dcf, loadtime - * **page cycles**: 25 - * **page timeout**: 90000 - * **playback**: mitmproxy - * **playback pageset manifest**: mitm6-windows-firefox-reddit-billgates-post.manifest - * **playback version**: 6.0.2 - * **test cmds**: ["measure.start", "billg"], ["navigate", "https://www.reddit.com/user/thisisbillgates/"], ["wait.byTime", 5000], ["measure.stop", ""], ["measure.start", "posts"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "comments"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "hot"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], ["wait.byTime", 15000], ["measure.stop", ""], ["measure.start", "top"], ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], ["wait.byTime", 15000], ["measure.stop", ""], - * **test url**: ``__ - * **type**: pageload - * **unit**: ms - * **use live sites**: false - - - Live ---- A set of test pages that are run as live sites instead of recorded versions. These tests are available on all browsers, on all platforms. (WX: WebExtension, BT: Browsertime, FF: Firefox, CH: Chrome, CU: Chromium, GV: Geckoview, RB: Refbrow, FE: Fenix, CH-M: Chrome mobile) diff --git a/testing/performance/hooks_recording.py b/testing/performance/hooks_recording.py index 0844753484d0..736b909d9422 100644 --- a/testing/performance/hooks_recording.py +++ b/testing/performance/hooks_recording.py @@ -7,8 +7,6 @@ import json import platform from pathlib import Path -from manifestparser.util import evaluate_list_from_string - from mozperftest.test.browsertime import add_options, add_option options = [ @@ -82,9 +80,4 @@ def before_runs(env): add_option(env, "browsertime.screenshot", "true") add_option(env, "browsertime.testName", test_site.get("name")) - # pack array into string for transport to javascript, is there a better way? - cmds = evaluate_list_from_string(test_site.get("test_cmds", "[]")) - parsed_cmds = [":::".join([str(i) for i in item]) for item in cmds if item] - add_option(env, "browsertime.commands", ";;;".join(parsed_cmds)) - print("Recording %s to file: %s" % (test_site.get("url"), recording_file)) diff --git a/testing/performance/pageload_sites.json b/testing/performance/pageload_sites.json index b212820b9a2b..ec99bd2f4ee3 100644 --- a/testing/performance/pageload_sites.json +++ b/testing/performance/pageload_sites.json @@ -314,80 +314,6 @@ "login": false, "name": "youtube", "test_url": "https://www.youtube.com" - }, - { - "login": false, - "name": "cnn-nav", - "test_url": "https://www.cnn.com/", - "test_cmds": [ - ["measure.start", "landing"], - ["navigate", "https://www.cnn.com"], - ["wait.byTime", 4000], - ["measure.stop", ""], - ["measure.start", "world"], - ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], - ["wait.byTime", 1000], - ["measure.stop", ""] - ] - }, - { - "login": false, - "name": "reddit-billgates-ama", - "test_url": "https://www.reddit.com/", - "test_cmds": [ - ["measure.start", "billg-ama"], - ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "members"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], - ["wait.byTime", 1000], - ["measure.stop", ""] - ] - }, - { - "login": false, - "name": "reddit-billgates-post", - "test_url": "https://www.reddit.com/user/thisisbillgates/", - "test_cmds": [ - ["measure.start", "billg"], - ["navigate", "https://www.reddit.com/user/thisisbillgates/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "posts"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "comments"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "hot"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "top"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], - ["wait.byTime", 15000], - ["measure.stop", ""] - ] - }, - { - "login": false, - "name": "facebook-nav", - "test_url": "https://www.facebook.com/", - "test_cmds": [ - ["navigate", "https://www.facebook.com/login"], - ["wait.byTime", 30000], - ["measure.start", "https://www.facebook.com/"], - ["wait.byTime", 5000], - ["measure.start", "https://www.facebook.com/marketplace/"], - ["wait.byTime", 5000], - ["measure.start", "https://www.facebook.com/groups/discover/"], - ["wait.byTime", 10000], - ["measure.start", "https://www.facebook.com/friends"], - ["wait.byTime", 10000] - ] } ] } diff --git a/testing/performance/perftest_record.js b/testing/performance/perftest_record.js index 747d2bffee3f..e136c2f90e17 100644 --- a/testing/performance/perftest_record.js +++ b/testing/performance/perftest_record.js @@ -4,7 +4,7 @@ /* eslint-env node */ "use strict"; -async function pageload_test(context, commands) { +async function test(context, commands) { let testUrl = context.options.browsertime.url; let secondaryUrl = context.options.browsertime.secondary_url; let testName = context.options.browsertime.testName; @@ -25,83 +25,6 @@ async function pageload_test(context, commands) { // Wait for browser to settle await commands.wait.byTime(1000); -} - -async function get_command_function(cmd, commands) { - /* - Converts a string such as `measure.start` into the actual - function that is found in the `commands` module. - - XXX: Find a way to share this function between - perftest_record.js and browsertime_interactive.js - */ - if (cmd == "") { - throw new Error("A blank command was given."); - } else if (cmd.endsWith(".")) { - throw new Error( - "An extra `.` was found at the end of this command: " + cmd - ); - } - - // `func` will hold the actual method that needs to be called, - // and the `parent_mod` is the context required to run the `func` - // method. Without that context, `this` becomes undefined in the browsertime - // classes. - let func = null; - let parent_mod = null; - for (let func_part of cmd.split(".")) { - if (func_part == "") { - throw new Error( - "An empty function part was found in the command: " + cmd - ); - } - - if (func === null) { - parent_mod = commands; - func = commands[func_part]; - } else if (func !== undefined) { - parent_mod = func; - func = func[func_part]; - } else { - break; - } - } - - if (func == undefined) { - throw new Error( - "The given command could not be found as a function: " + cmd - ); - } - - return [func, parent_mod]; -} - -async function interactive_test(input_cmds, context, commands) { - let cmds = input_cmds.split(";;;"); - - await commands.navigate("about:blank"); - - for (let cmdstr of cmds) { - let [cmd, ...args] = cmdstr.split(":::"); - let [func, parent_mod] = await get_command_function(cmd, commands); - - try { - await func.call(parent_mod, ...args); - } catch (e) { - context.log.info( - `Exception found while running \`commands.${cmd}(${args})\`: ` + e - ); - } - } -} - -async function test(context, commands) { - let input_cmds = context.options.browsertime.commands; - if (input_cmds !== undefined) { - await interactive_test(input_cmds, context, commands); - } else { - await pageload_test(context, commands); - } return true; } diff --git a/testing/raptor/browsertime/browsertime_interactive.js b/testing/raptor/browsertime/browsertime_interactive.js deleted file mode 100644 index e8ce2ddd956a..000000000000 --- a/testing/raptor/browsertime/browsertime_interactive.js +++ /dev/null @@ -1,104 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/* eslint-env node */ - -async function get_command_function(cmd, commands) { - /* - Converts a string such as `measure.start` into the actual - function that is found in the `commands` module. - - XXX: Find a way to share this function between - perftest_record.js and browsertime_interactive.js - */ - if (cmd == "") { - throw new Error("A blank command was given."); - } else if (cmd.endsWith(".")) { - throw new Error( - "An extra `.` was found at the end of this command: " + cmd - ); - } - - // `func` will hold the actual method that needs to be called, - // and the `parent_mod` is the context required to run the `func` - // method. Without that context, `this` becomes undefined in the browsertime - // classes. - let func = null; - let parent_mod = null; - for (let func_part of cmd.split(".")) { - if (func_part == "") { - throw new Error( - "An empty function part was found in the command: " + cmd - ); - } - - if (func === null) { - parent_mod = commands; - func = commands[func_part]; - } else if (func !== undefined) { - parent_mod = func; - func = func[func_part]; - } else { - break; - } - } - - if (func == undefined) { - throw new Error( - "The given command could not be found as a function: " + cmd - ); - } - - return [func, parent_mod]; -} - -module.exports = async function(context, commands) { - context.log.info("Starting an interactive browsertime test"); - let page_cycles = context.options.browsertime.page_cycles; - let post_startup_delay = context.options.browsertime.post_startup_delay; - let input_cmds = context.options.browsertime.commands; - - context.log.info( - "Waiting for %d ms (post_startup_delay)", - post_startup_delay - ); - await commands.wait.byTime(post_startup_delay); - - // unpack commands from python - let cmds = input_cmds.split(";;;"); - - // let pages_visited = 0; - for (let count = 0; count < page_cycles; count++) { - context.log.info("Navigating to about:blank w/nav, count: " + count); - await commands.navigate("about:blank"); - - let pages_visited = []; - for (let cmdstr of cmds) { - let [cmd, ...args] = cmdstr.split(":::"); - - if (cmd == "measure.start") { - if (args[0] != "") { - pages_visited.push(args[0]); - } - } - - let [func, parent_mod] = await get_command_function(cmd, commands); - - try { - await func.call(parent_mod, ...args); - } catch (e) { - context.log.info( - `Exception found while running \`commands.${cmd}(${args})\`: ` - ); - context.log.info(e.stack); - } - } - - // Log the number of pages visited for results parsing - context.log.info("[] metrics: pages_visited: " + pages_visited); - } - - context.log.info("Browsertime pageload ended."); - return true; -}; diff --git a/testing/raptor/browsertime/browsertime_pageload.js b/testing/raptor/browsertime/browsertime_pageload.js index 9b8f3520a2ee..51a5c624b1b2 100644 --- a/testing/raptor/browsertime/browsertime_pageload.js +++ b/testing/raptor/browsertime/browsertime_pageload.js @@ -23,7 +23,7 @@ module.exports = async function(context, commands) { context.log.info("Navigating to secondary url:" + secondary_url); await commands.navigate(secondary_url); } else { - context.log.info("Navigating to about:blank, count: " + count); + context.log.info("Navigating to about:blank"); await commands.navigate("about:blank"); } @@ -34,7 +34,6 @@ module.exports = async function(context, commands) { context.log.info("Cycle %d, starting the measure", count); await commands.measure.start(test_url); } - context.log.info("Browsertime pageload ended."); return true; }; diff --git a/testing/raptor/raptor/browsertime/base.py b/testing/raptor/raptor/browsertime/base.py index 154278a6a2e6..3a8c3c10ea8a 100644 --- a/testing/raptor/raptor/browsertime/base.py +++ b/testing/raptor/raptor/browsertime/base.py @@ -15,7 +15,6 @@ import six import sys import mozprocess -from manifestparser.util import evaluate_list_from_string from benchmark import Benchmark from logger.logger import RaptorLogger from perftest import Perftest @@ -42,8 +41,6 @@ class Browsertime(Perftest): def __init__(self, app, binary, process_handler=None, **kwargs): self.browsertime = True self.browsertime_failure = "" - self.page_count = [] - self.process_handler = process_handler or mozprocess.ProcessHandler for key in list(kwargs): if key.startswith("browsertime_"): @@ -203,26 +200,15 @@ class Browsertime(Perftest): ] else: # Custom scripts are treated as pageload tests for now - if test.get("interactive", False): - browsertime_script = [ - os.path.join( - os.path.dirname(__file__), - "..", - "..", - "browsertime", - "browsertime_interactive.js", - ) - ] - else: - browsertime_script = [ - os.path.join( - os.path.dirname(__file__), - "..", - "..", - "browsertime", - test.get("test_script", "browsertime_pageload.js"), - ) - ] + browsertime_script = [ + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "browsertime", + test.get("test_script", "browsertime_pageload.js"), + ) + ] btime_args = self.browsertime_args if self.config["app"] in ("chrome", "chromium", "chrome-m"): @@ -283,11 +269,6 @@ class Browsertime(Perftest): for var, val in self.config.get("environment", {}).items(): browsertime_options.extend(["--firefox.env", "{}={}".format(var, val)]) - # Parse the test commands (if any) from the test manifest - cmds = evaluate_list_from_string(test.get("test_cmds", "[]")) - parsed_cmds = [":::".join([str(i) for i in item]) for item in cmds if item] - browsertime_options.extend(["--browsertime.commands", ";;;".join(parsed_cmds)]) - if self.verbose: browsertime_options.append("-vvv") @@ -493,9 +474,6 @@ class Browsertime(Perftest): proc.kill() elif "warning" in level: LOG.warning(msg) - elif "metrics" in level: - vals = msg.split(":")[-1].strip() - self.page_count = vals.split(",") else: LOG.info(msg) diff --git a/testing/raptor/raptor/manifest.py b/testing/raptor/raptor/manifest.py index 177a603a92c4..5e0117d0904f 100644 --- a/testing/raptor/raptor/manifest.py +++ b/testing/raptor/raptor/manifest.py @@ -563,15 +563,18 @@ def get_raptor_test_list(args, oskey): # remove the 'hero =' line since no longer measuring hero del next_test["hero"] - bool_settings = [ - "lower_is_better", - "subtest_lower_is_better", - "accept_zero_vismet", - "interactive", - ] - for setting in bool_settings: - if next_test.get(setting, None) is not None: - next_test[setting] = bool_from_str(next_test.get(setting)) + if next_test.get("lower_is_better") is not None: + next_test["lower_is_better"] = bool_from_str( + next_test.get("lower_is_better") + ) + if next_test.get("subtest_lower_is_better") is not None: + next_test["subtest_lower_is_better"] = bool_from_str( + next_test.get("subtest_lower_is_better") + ) + if next_test.get("accept_zero_vismet", None) is not None: + next_test["accept_zero_vismet"] = bool_from_str( + next_test.get("accept_zero_vismet") + ) # write out .json test setting files for the control server to read and send to web ext if len(tests_to_run) != 0: diff --git a/testing/raptor/raptor/output.py b/testing/raptor/raptor/output.py index 7a16675b48b6..32ee19b1811d 100644 --- a/testing/raptor/raptor/output.py +++ b/testing/raptor/raptor/output.py @@ -191,23 +191,11 @@ class PerftestOutput(object): if self.summarized_results == {}: success = False LOG.error( - "no summarized raptor results found for any of %s" - % ", ".join(test_names) + "no summarized raptor results found for %s" % ", ".join(test_names) ) else: for suite in self.summarized_results["suites"]: tname = suite["name"] - - # as we do navigation, tname could end in . - # test_names doesn't have tname, so either add it to test_names, - # or strip it - parts = tname.split(".") - try: - tname = ".".join(parts[:-1]) - except Exception as e: - LOG.info("no alias found on test, ignoring: %s" % e) - pass - # Since test names might have been modified, check if # part of the test name exists in the test_names list entries found = False @@ -217,7 +205,7 @@ class PerftestOutput(object): break if not found: success = False - LOG.error("no summarized raptor results found for %s" % (tname)) + LOG.error("no summarized raptor results found for %s" % tname) with open(results_path, "w") as f: for result in self.summarized_results: diff --git a/testing/raptor/raptor/perfdocs/config.yml b/testing/raptor/raptor/perfdocs/config.yml index 89ebd248653d..1e1ec3aaff76 100644 --- a/testing/raptor/raptor/perfdocs/config.yml +++ b/testing/raptor/raptor/perfdocs/config.yml @@ -152,13 +152,6 @@ suites: tests: process-switch: "Measures process switch time" welcome: "Measures pageload metrics for the first-install about:welcome page" - interactive: - description: "Browsertime tests that interact with the webpage. Includes responsiveness tests as they make use of this support for navigation. These form of tests allow the specification of browsertime commands through the test manifest." - tests: - cnn-nav: "Navigates to cnn main page, then to the world sub-page." - facebook-nav: "Navigates to facebook, then the sub-pages friends, marketplace, groups." - reddit-billgates-ama: "Navigates from the Bill Gates AMA to the Reddit members section." - reddit-billgates-post: "Navigates the `thisisbillgates` user starting at the main user page, then to the posts, comments, hot, and top sections." unittests: description: "These tests aren't used in standard testing, they are only used in the Raptor unit tests (they are similar to raptor-tp6 tests though)." tests: diff --git a/testing/raptor/raptor/perftest.py b/testing/raptor/raptor/perftest.py index 78ca31c29fbc..342cb916dcf4 100644 --- a/testing/raptor/raptor/perftest.py +++ b/testing/raptor/raptor/perftest.py @@ -483,7 +483,6 @@ class Perftest(object): self.config["raptor_json_path"] = raptor_json_path self.config["artifact_dir"] = self.artifact_dir - self.config["page_count"] = self.page_count res = self.results_handler.summarize_and_output(self.config, tests, test_names) # gecko profiling symbolication diff --git a/testing/raptor/raptor/raptor.ini b/testing/raptor/raptor/raptor.ini index 8f23c9493307..80826eb44ab7 100644 --- a/testing/raptor/raptor/raptor.ini +++ b/testing/raptor/raptor/raptor.ini @@ -47,6 +47,3 @@ # First-install pageload test [include:tests/custom/browsertime-welcome.ini] - -# Interactive raptor-browsertime tests -[include:tests/interactive/browsertime-responsiveness.ini] diff --git a/testing/raptor/raptor/results.py b/testing/raptor/raptor/results.py index 573796ab110e..33dfbd5689ba 100644 --- a/testing/raptor/raptor/results.py +++ b/testing/raptor/raptor/results.py @@ -338,14 +338,7 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): pass def parse_browsertime_json( - self, - raw_btresults, - page_cycles, - cold, - browser_cycles, - measure, - page_count, - test_name, + self, raw_btresults, page_cycles, cold, browser_cycles, measure ): """ Receive a json blob that contains the results direct from the browsertime tool. Parse @@ -510,7 +503,6 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): ) # now parse out the values - page_counter = 0 for raw_result in raw_btresults: if not raw_result["browserScripts"]: raise MissingResultsError("Browsertime cycle produced no measurements.") @@ -521,20 +513,11 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): # Desktop chrome doesn't have `browser` scripts data available for now bt_browser = raw_result["browserScripts"][0].get("browser", None) bt_ver = raw_result["info"]["browsertime"]["version"] - - # when doing actions, we append a .X for each additional pageload in a scenario - extra = "" - if len(page_count) > 0: - extra = ".%s" % page_count[page_counter % len(page_count)] - url_parts = raw_result["info"]["url"].split("/") - page_counter += 1 - - bt_url = "%s%s/%s," % ("/".join(url_parts[:-1]), extra, url_parts[-1]) + bt_url = (raw_result["info"]["url"],) bt_result = { "bt_ver": bt_ver, "browser": bt_browser, "url": bt_url, - "name": "%s%s" % (test_name, extra), "measurements": {}, "statistics": {}, } @@ -565,7 +548,6 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): "bt_ver": bt_ver, "browser": bt_browser, "url": bt_url, - "name": "%s%s" % (test_name, extra), "measurements": {}, "statistics": {}, } @@ -769,8 +751,6 @@ class BrowsertimeResultsHandler(PerftestResultsHandler): test["cold"], test["browser_cycles"], test.get("measure"), - test_config.get("page_count", []), - test["name"], ): def _new_standard_result(new_result, subtest_unit="ms"): diff --git a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini b/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini deleted file mode 100644 index 2b2de1a3c370..000000000000 --- a/testing/raptor/raptor/tests/interactive/browsertime-responsiveness.ini +++ /dev/null @@ -1,101 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Raptor-browsertime interactive responsiveness tests - -[DEFAULT] -accept_zero_vismet = true -alert_on = fcp, loadtime -alert_threshold = 2.0 -apps = firefox, chrome, chromium -browser_cycles = 25 -gecko_profile_entries = 14000000 -gecko_profile_interval = 1 -interactive = true -lower_is_better = true -measure = fnbpaint, fcp, dcf, loadtime -page_cycles = 25 -page_timeout = 60000 -playback = mitmproxy -playback_version = 6.0.2 -type = pageload -unit = ms -use_live_sites = false - -# Keep this list in alphabetical order - -[cnn-nav] -test_url = https://www.cnn.com/ -playback_pageset_manifest = mitm6-windows-firefox-cnn-nav.manifest -browser_cycles = 10 # used with --cold -test_cmds = - ["measure.start", "landing"], - ["navigate", "https://www.cnn.com"], - ["wait.byTime", 4000], - ["measure.stop", ""], - ["measure.start", "world"], - ["click.byXpathAndWait", "/html/body/div[5]/div/div/header/div/div[1]/div/div[2]/nav/ul/li[2]/a"], - ["wait.byTime", 1000], - ["measure.stop", ""], - -[facebook-nav] -test_url = https://www.facebook.com -playback_pageset_manifest = mitm6-windows-firefox-facebook-nav.manifest -page_timeout = 90000 -test_cmds = - ["measure.start", "landing"], - ["navigate", "https://www.facebook.com/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "friends"], - ["navigate", "https://www.facebook.com/friends/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "marketplace"], - ["navigate", "https://www.facebook.com/marketplace"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "groups"], - ["navigate", "https://www.facebook.com/groups/discover/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - -[reddit-billgates-ama] -test_url = https://www.reddit.com/ -playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-ama.manifest -test_cmds = - ["measure.start", "billg-ama"], - ["navigate", "https://www.reddit.com/r/IAmA/comments/m8n4vt/im_bill_gates_cochair_of_the_bill_and_melinda/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "members"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div[3]/div[2]/div/div[1]/div/div[4]/div[1]/div"], - ["wait.byTime", 1000], - ["measure.stop", ""], - -[reddit-billgates-post] -test_url = https://www.reddit.com/user/thisisbillgates/ -playback_pageset_manifest = mitm6-windows-firefox-reddit-billgates-post.manifest -page_timeout = 90000 -test_cmds = - ["measure.start", "billg"], - ["navigate", "https://www.reddit.com/user/thisisbillgates/"], - ["wait.byTime", 5000], - ["measure.stop", ""], - ["measure.start", "posts"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[2]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "comments"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[2]/div/div/div/a[3]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "hot"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[2]"], - ["wait.byTime", 15000], - ["measure.stop", ""], - ["measure.start", "top"], - ["click.byXpathAndWait", "/html/body/div[1]/div/div[2]/div[2]/div/div/div/div[2]/div[4]/div[1]/div[1]/div[2]/a[3]"], - ["wait.byTime", 15000], - ["measure.stop", ""], diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest deleted file mode 100644 index b68277f26917..000000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-cnn-nav.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm6-windows-firefox-cnn-nav.zip", - "size": 11461220, - "algorithm": "sha512", - "digest": "201615126b59bd08df0a85226e6ee1aa3409e9b99b028b59ae61cce66fba05cbb7d33a7b2d2997370b4215325931b30bd6190023855c6d0d2415a9fdcba77c77", - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest deleted file mode 100644 index e44c47236ed8..000000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-facebook-nav.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm6-windows-firefox-facbook-nav.zip", - "size": 30903320, - "algorithm": "sha512", - "digest": "5d565ce257c61fba09f382298aa4709d5373ee0f59e96f6313a0863cd51ce07fbbfbbbe7659e358fab4a1dcd2927838fbf65427ff20e900dc921d48ee370af83", - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest deleted file mode 100644 index 635fbff4943d..000000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-ama.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm6-windows-firefox-reddit-billgates-ama.zip", - "size": 10560692, - "algorithm": "sha512", - "digest": "77648b7d0499c97e7b4f48a8861566ab3c66c7b74ae5b842d62c59fbdb782332cd8b830a8bbf3645904772c7e43b98aaa4d807ae9150a4be9da137b768fa67aa", - "visibility": "public" - } -] diff --git a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest b/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest deleted file mode 100644 index c149ffd8e800..000000000000 --- a/testing/raptor/raptor/tooltool-manifests/playback/mitm6-windows-firefox-reddit-billgates-post.manifest +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "filename": "mitm6-windows-firefox-reddit-billgates-post.zip", - "size": 6400866, - "algorithm": "sha512", - "digest": "2e70c78430ab8e6cef1ada7bb5638a8321bbd5c27880761311144e559b90cf3b49b98b6ce69c88ead1b9a841f8a3b0955dcb0c9f9910690b595c59b00e841b72", - "visibility": "public" - } -]