From bb49296ca2998a2b01fae581342ea8511db249e6 Mon Sep 17 00:00:00 2001 From: afinder Date: Wed, 24 Nov 2021 14:57:28 +0000 Subject: [PATCH] Bug 1735324 [perfdocs] Enumerate AWSY tests using PerfDocs r=perftest-reviewers,sparky,AlexandruIonescu Differential Revision: https://phabricator.services.mozilla.com/D128198 --- testing/awsy/perfdocs/config.yaml | 11 +- testing/awsy/perfdocs/index.rst | 5 +- testing/perfdocs/generated/awsy.rst | 157 ++++++++++++++++++++- tools/lint/perfdocs/framework_gatherers.py | 78 +++++++++- tools/lint/perfdocs/gatherer.py | 2 + tools/lint/test/test_perfdocs.py | 9 +- 6 files changed, 254 insertions(+), 8 deletions(-) diff --git a/testing/awsy/perfdocs/config.yaml b/testing/awsy/perfdocs/config.yaml index bae393325692..50993d6b7c60 100644 --- a/testing/awsy/perfdocs/config.yaml +++ b/testing/awsy/perfdocs/config.yaml @@ -4,5 +4,12 @@ --- name: awsy manifest: None -static-only: True -suites: {} +static-only: False +suites: + Awsy tests: + description: "" + tests: + base: "FF" + dmd: "FF" + tp5: "FF" + tp6: "FF" diff --git a/testing/awsy/perfdocs/index.rst b/testing/awsy/perfdocs/index.rst index e7f0a813b908..e0b0d5ac0dee 100644 --- a/testing/awsy/perfdocs/index.rst +++ b/testing/awsy/perfdocs/index.rst @@ -1,5 +1,6 @@ +==== AWSY -##### +==== Are We Slim Yet project (commonly known as AWSY) tracks memory usage across builds. @@ -9,6 +10,8 @@ AWSY tests consist of three types: TP5*, TP6, and Base Memory Usage. *\*TP5 tests are out of date and no longer maintained. These tests are scheduled to be removed:* https://bugzilla.mozilla.org/show_bug.cgi?id=1712406 +{documentation} + Running AWSY Locally ********************* diff --git a/testing/perfdocs/generated/awsy.rst b/testing/perfdocs/generated/awsy.rst index e7f0a813b908..0471cfe3ea9f 100644 --- a/testing/perfdocs/generated/awsy.rst +++ b/testing/perfdocs/generated/awsy.rst @@ -1,5 +1,6 @@ +==== AWSY -##### +==== Are We Slim Yet project (commonly known as AWSY) tracks memory usage across builds. @@ -9,6 +10,160 @@ AWSY tests consist of three types: TP5*, TP6, and Base Memory Usage. *\*TP5 tests are out of date and no longer maintained. These tests are scheduled to be removed:* https://bugzilla.mozilla.org/show_bug.cgi?id=1712406 +Awsy tests +---------- + + +.. dropdown:: base (FF) + :container: + anchor-id-base-Awsy-tests + + * **Test Task**: + * test-linux1804-64-clang-trunk-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: None + * awsy-base-fis-e10s: None + + * test-linux1804-64-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: None + * awsy-base-fis-e10s: None + + * test-linux1804-64-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + * test-macosx1015-64-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + * test-windows10-32-2004-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: None + * awsy-base-fis-e10s: None + + * test-windows10-32-2004-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: None + * awsy-base-fis-e10s: None + + * test-windows10-64-2004-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: None + * awsy-base-fis-e10s: None + + * test-windows10-64-2004-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-base-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-base-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + +.. dropdown:: dmd (FF) + :container: + anchor-id-dmd-Awsy-tests + + * **Test Task**: + * test-linux1804-64-clang-trunk-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-linux1804-64-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-linux1804-64-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-macosx1015-64-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-windows10-32-2004-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-windows10-32-2004-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-windows10-64-2004-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + * test-windows10-64-2004-shippable-qr/opt + * awsy-base-dmd-e10s: None + * awsy-dmd-e10s: None + + +.. dropdown:: tp5 (FF) + :container: + anchor-id-tp5-Awsy-tests + + * **Test Task**: + * test-linux1804-64-clang-trunk-qr/opt + * awsy-e10s: None + + * test-linux1804-64-qr/opt + * awsy-e10s: None + + * test-linux1804-64-shippable-qr/opt + * awsy-e10s: None + + * test-macosx1015-64-shippable-qr/opt + * awsy-e10s: None + + * test-windows10-32-2004-qr/opt + * awsy-e10s: None + + * test-windows10-32-2004-shippable-qr/opt + * awsy-e10s: None + + * test-windows10-64-2004-qr/opt + * awsy-e10s: None + + * test-windows10-64-2004-shippable-qr/opt + * awsy-e10s: None + + +.. dropdown:: tp6 (FF) + :container: + anchor-id-tp6-Awsy-tests + + * **Test Task**: + * test-linux1804-64-clang-trunk-qr/opt + * awsy-tp6-e10s: None + * awsy-tp6-fis-e10s: None + + * test-linux1804-64-qr/opt + * awsy-tp6-e10s: None + * awsy-tp6-fis-e10s: None + + * test-linux1804-64-shippable-qr/opt + * awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + * test-macosx1015-64-shippable-qr/opt + * awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + * test-windows10-32-2004-qr/opt + * awsy-tp6-e10s: None + * awsy-tp6-fis-e10s: None + + * test-windows10-32-2004-shippable-qr/opt + * awsy-tp6-e10s: None + * awsy-tp6-fis-e10s: None + + * test-windows10-64-2004-qr/opt + * awsy-tp6-e10s: None + * awsy-tp6-fis-e10s: None + + * test-windows10-64-2004-shippable-qr/opt + * awsy-tp6-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + * awsy-tp6-fis-e10s: integration, mozilla-beta, mozilla-central, mozilla-release + + + + Running AWSY Locally ********************* diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py index f3cbadb51e3f..3983a537b203 100644 --- a/tools/lint/perfdocs/framework_gatherers.py +++ b/tools/lint/perfdocs/framework_gatherers.py @@ -40,6 +40,10 @@ class FrameworkGatherer(object): self._manifest = None self.script_infos = {} self._task_list = {} + self._task_match_pattern = re.compile(r"([\w\W]*/[pgo|opt]*)-([\w\W]*)") + + def get_task_match(self, task_name): + return re.search(self._task_match_pattern, task_name) def get_manifest_path(self): """ @@ -135,7 +139,7 @@ class RaptorGatherer(FrameworkGatherer): run_on_projects = self._taskgraph[task].attributes["run_on_projects"] test_match = re.search(r"[\s']--test[\s=](.+?)[\s']", str(command)) - task_match = re.search(r"([\w\W]*/[pgo|opt]*)-([\w\W]*)", task) + task_match = self.get_task_match(task) if test_match and task_match: test = test_match.group(1) platform = task_match.group(1) @@ -376,6 +380,78 @@ class TalosGatherer(FrameworkGatherer): return self._build_section_with_header(title, content, header_type="H3") +class AwsyGatherer(FrameworkGatherer): + """ + Gatherer for the Awsy framework. + """ + + def _generate_ci_tasks(self): + for task_name in self._taskgraph.keys(): + task = self._taskgraph[task_name] + + if type(task) == dict: + awsy_test = task["task"]["extra"].get("suite", []) + run_on_projects = task["attributes"]["run_on_projects"] + else: + awsy_test = task.task["extra"].get("suite", []) + run_on_projects = task.attributes["run_on_projects"] + + task_match = self.get_task_match(task_name) + + if "awsy" in awsy_test and task_match: + platform = task_match.group(1) + test_name = task_match.group(2) + item = {"test_name": test_name, "run_on_projects": run_on_projects} + self._task_list.setdefault(platform, []).append(item) + + def get_suite_list(self): + self._suite_list = {"Awsy tests": ["tp6", "base", "dmd", "tp5"]} + return self._suite_list + + def get_test_list(self): + self._generate_ci_tasks() + return { + "Awsy tests": { + "tp6": "", + "base": "", + "dmd": "", + "tp5": "", + } + } + + def build_suite_section(self, title, content): + return self._build_section_with_header( + title.capitalize(), content, header_type="H4" + ) + + def build_test_description(self, title, test_description="", suite_name=""): + dropdown_suite_name = suite_name.replace(" ", "-") + result = f".. dropdown:: {title} ({test_description})\n" + result += f" :container: + anchor-id-{title}-{dropdown_suite_name}\n\n" + result += " * **Test Task**:\n" + + # tp5 tests are represented by awsy-e10s test names + # while the others have their title in test names + search_tag = "awsy-e10s" if title == "tp5" else title + for platform in sorted(self._task_list.keys()): + result += f" * {platform}\n" + for test_dict in sorted( + self._task_list[platform], key=lambda d: d["test_name"] + ): + if search_tag in test_dict["test_name"]: + run_on_project = ": " + ( + ", ".join(test_dict["run_on_projects"]) + if test_dict["run_on_projects"] + else "None" + ) + result += ( + f" * {test_dict['test_name']}{run_on_project}\n" + ) + result += "\n" + + return [result] + + class StaticGatherer(FrameworkGatherer): """ A noop gatherer for frameworks with static-only documentation. diff --git a/tools/lint/perfdocs/gatherer.py b/tools/lint/perfdocs/gatherer.py index 6d1404cc23ee..bbdacf279c6f 100644 --- a/tools/lint/perfdocs/gatherer.py +++ b/tools/lint/perfdocs/gatherer.py @@ -13,6 +13,7 @@ from perfdocs.framework_gatherers import ( RaptorGatherer, StaticGatherer, TalosGatherer, + AwsyGatherer, ) logger = PerfDocLogger() @@ -22,6 +23,7 @@ frameworks = { "raptor": RaptorGatherer, "mozperftest": MozperftestGatherer, "talos": TalosGatherer, + "awsy": AwsyGatherer, } diff --git a/tools/lint/test/test_perfdocs.py b/tools/lint/test/test_perfdocs.py index 71a4250eb55d..0ef46a5f7c69 100644 --- a/tools/lint/test/test_perfdocs.py +++ b/tools/lint/test/test_perfdocs.py @@ -53,7 +53,6 @@ module.exports = { }; """ - SAMPLE_CONFIG = """ name: mozperftest manifest: None @@ -65,7 +64,6 @@ suites: Example: "" """ - DYNAMIC_SAMPLE_CONFIG = """ name: {} manifest: None @@ -81,7 +79,6 @@ suites: Example: "Performance test Example from another_suite." """ - SAMPLE_INI = """ [Example] test_url = Example_url @@ -487,6 +484,12 @@ def test_perfdocs_framework_gatherers(logger, structured_logger, perfdocs_sample assert suitetests continue + if framework == "awsy": + for suite, suitetests in fg.get_test_list().items(): + assert suite == "Awsy tests" + assert suitetests + continue + for suite, suitetests in fg.get_test_list().items(): assert suite == "suite" for test, manifest in suitetests.items():