diff --git a/tools/lint/perfdocs/__init__.py b/tools/lint/perfdocs/__init__.py new file mode 100644 index 000000000000..fe19cabad18e --- /dev/null +++ b/tools/lint/perfdocs/__init__.py @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import os + +from perfdocs import perfdocs +from mozlint.util import pip + +here = os.path.abspath(os.path.dirname(__file__)) +PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, 'requirements.txt') + + +def setup(root, **lintargs): + if not pip.reinstall_program(PERFDOCS_REQUIREMENTS_PATH): + print("Cannot install requirements.") + return 1 + + +def lint(paths, config, logger, fix=None, **lintargs): + return perfdocs.run_perfdocs( + config, logger=logger, paths=paths, verify=True + ) diff --git a/tools/lint/perfdocs/framework_gatherers.py b/tools/lint/perfdocs/framework_gatherers.py new file mode 100644 index 000000000000..7588e4e01980 --- /dev/null +++ b/tools/lint/perfdocs/framework_gatherers.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import os +import re + +from perfdocs.utils import read_yaml +from manifestparser import TestManifest + +''' +This file is for framework specific gatherers since manifests +might be parsed differently in each of them. The gatherers +must implement the FrameworkGatherer class. +''' + + +class FrameworkGatherer(object): + ''' + Abstract class for framework gatherers. + ''' + + def __init__(self, yaml_path, workspace_dir): + ''' + Generic initialization for a framework gatherer. + ''' + self.workspace_dir = workspace_dir + self._yaml_path = yaml_path + self._suite_list = {} + self._manifest_path = '' + self._manifest = None + + def get_manifest_path(self): + ''' + Returns the path to the manifest based on the + manifest entry in the frameworks YAML configuration + file. + + :return str: Path to the manifest. + ''' + if self._manifest_path: + return self._manifest_path + + yaml_content = read_yaml(self._yaml_path) + self._manifest_path = os.path.join( + self.workspace_dir, yaml_content["manifest"] + ) + return self._manifest_path + + def get_suite_list(self): + ''' + Each framework gatherer must return a dictionary with + the following structure. Note that the test names must + be relative paths so that issues can be correctly issued + by the reviewbot. + + :return dict: A dictionary with the following structure: { + "suite_name": [ + 'testing/raptor/test1', + 'testing/raptor/test2' + ] + } + ''' + raise NotImplementedError + + +class RaptorGatherer(FrameworkGatherer): + ''' + Gatherer for the Raptor framework. + ''' + + def get_suite_list(self): + ''' + Returns a dictionary containing a mapping from suites + to the tests they contain. + + :return dict: A dictionary with the following structure: { + "suite_name": [ + 'testing/raptor/test1', + 'testing/raptor/test2' + ] + } + ''' + if self._suite_list: + return self._suite_list + + manifest_path = self.get_manifest_path() + + # Get the tests from the manifest + test_manifest = TestManifest([manifest_path], strict=False) + test_list = test_manifest.active_tests(exists=False, disabled=False) + + # Parse the tests into the expected dictionary + for test in test_list: + # Get the top-level suite + s = os.path.basename(test["here"]) + if s not in self._suite_list: + self._suite_list[s] = [] + + # Get the individual test + fpath = re.sub(".*testing", "testing", test['manifest']) + + if fpath not in self._suite_list[s]: + self._suite_list[s].append(fpath) + + return self._suite_list diff --git a/tools/lint/perfdocs/gatherer.py b/tools/lint/perfdocs/gatherer.py new file mode 100644 index 000000000000..e9ee8713dd79 --- /dev/null +++ b/tools/lint/perfdocs/gatherer.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import os +import re + +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import read_yaml +from perfdocs.framework_gatherers import RaptorGatherer + +logger = PerfDocLogger() + +# TODO: Implement decorator/searcher to find the classes. +frameworks = { + "raptor": RaptorGatherer, +} + + +class Gatherer(object): + ''' + Gatherer produces the tree of the perfdoc's entries found + and can obtain manifest-based test lists. Used by the Verifier. + ''' + + def __init__(self, root_dir, workspace_dir): + ''' + Initialzie the Gatherer. + + :param str root_dir: Path to the testing directory. + :param str workspace_dir: Path to the gecko checkout. + ''' + self.root_dir = root_dir + self.workspace_dir = workspace_dir + self._perfdocs_tree = [] + self._test_list = [] + + @property + def perfdocs_tree(self): + ''' + Returns the perfdocs_tree, and computes it + if it doesn't exist. + + :return dict: The perfdocs tree containing all + framework perfdoc entries. See `fetch_perfdocs_tree` + for information on the data strcture. + ''' + if self._perfdocs_tree: + return self.perfdocs_tree + else: + self.fetch_perfdocs_tree() + return self._perfdocs_tree + + def fetch_perfdocs_tree(self): + ''' + Creates the perfdocs tree with the following structure: + [ + { + "path": Path to the perfdocs directory. + "yml": Name of the configuration YAML file. + "rst": Name of the RST file. + }, ... + ] + + This method doesn't return anything. The result can be found in + the perfdocs_tree attribute. + ''' + yml_match = re.compile('^config.y(a)?ml$') + rst_match = re.compile('^index.rst$') + + for dirpath, dirname, files in os.walk(self.root_dir): + # Walk through the testing directory tree + if dirpath.endswith('/perfdocs'): + matched = {"path": dirpath, "yml": "", "rst": ""} + for file in files: + # Add the yml/rst file to its key if re finds the searched file + if re.search(yml_match, file): + matched["yml"] = re.search(yml_match, file).string + if re.search(rst_match, file): + matched["rst"] = re.search(rst_match, file).string + # Append to structdocs if all the searched files were found + if all(matched.values()): + self._perfdocs_tree.append(matched) + + logger.log("Found {} perfdocs directories in {}" + .format(len(self._perfdocs_tree), self.root_dir)) + + def get_test_list(self, sdt_entry): + ''' + Use a perfdocs_tree entry to find the test list for + the framework that was found. + + :return: A framework info dictionary with fields: { + 'yml_path': Path to YAML, + 'yml_content': Content of YAML, + 'name': Name of framework, + 'test_list': Test list found for the framework + } + ''' + + # If it was computed before, return it + yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml']) + for entry in self._test_list: + if entry['yml_path'] == yaml_path: + return entry + + # Set up framework entry with meta data + yaml_content = read_yaml(yaml_path) + framework = { + 'yml_content': yaml_content, + 'yml_path': yaml_path, + 'name': yaml_content["name"], + } + + # Get and then store the frameworks tests + framework_gatherer = frameworks[framework["name"]]( + framework["yml_path"], + self.workspace_dir + ) + framework["test_list"] = framework_gatherer.get_suite_list() + + self._test_list.append(framework) + return framework diff --git a/tools/lint/perfdocs/logger.py b/tools/lint/perfdocs/logger.py new file mode 100644 index 000000000000..2dc117ee96e3 --- /dev/null +++ b/tools/lint/perfdocs/logger.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import re + + +class PerfDocLogger(object): + ''' + Logger for the PerfDoc tooling. Handles the warnings by outputting + them into through the StructuredLogger provided by lint. + ''' + PATHS = [] + LOGGER = None + + def __init__(self): + '''Initializes the PerfDocLogger.''' + + # Set up class attributes for all logger instances + if not PerfDocLogger.LOGGER: + raise Exception( + "Missing linting LOGGER instance for PerfDocLogger initialization" + ) + if not PerfDocLogger.PATHS: + raise Exception( + "Missing PATHS for PerfDocLogger initialization" + ) + self.logger = PerfDocLogger.LOGGER + + def log(self, msg): + ''' + Log a message. + + :param str msg: Message to log. + ''' + self.logger.info(msg) + + def warning(self, msg, files): + ''' + Logs a validation warning message. The warning message is + used as the error message that is output in the reviewbot. + + :param str msg: Message to log, it's also used as the error message + for the issue that is output by the reviewbot. + :param list/str files: The file(s) that this warning is about. + ''' + if type(files) != list: + files = [files] + + # Add a reviewbot error for each file that is given + for file in files: + # Get a relative path (reviewbot can't handle absolute paths) + # TODO: Expand to outside of the testing directory + fpath = re.sub(".*testing", "testing", file) + + # Filter out any issues that do not relate to the paths + # that are being linted + for path in PerfDocLogger.PATHS: + if path not in file: + continue + + # Output error entry + self.logger.lint_error( + message=msg, + lineno=0, + column=None, + path=fpath, + linter='perfdocs', + rule="Flawless performance docs." + ) + + break diff --git a/tools/lint/perfdocs/perfdocs.py b/tools/lint/perfdocs/perfdocs.py new file mode 100644 index 000000000000..a5726d2246a3 --- /dev/null +++ b/tools/lint/perfdocs/perfdocs.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function + +import os +import re + + +def run_perfdocs(config, logger=None, paths=None, verify=True, generate=False): + ''' + Build up performance testing documentation dynamically by combining + text data from YAML files that reside in `perfdoc` folders + across the `testing` directory. Each directory is expected to have + an `index.rst` file along with `config.yml` YAMLs defining what needs + to be added to the documentation. + + The YAML must also define the name of the "framework" that should be + used in the main index.rst for the performance testing documentation. + + The testing documentation list will be ordered alphabetically once + it's produced (to avoid unwanted shifts because of unordered dicts + and path searching). + + Note that the suite name headings will be given the H4 (---) style so it + is suggested that you use H3 (===) style as the heading for your + test section. H5 will be used be used for individual tests within each + suite. + + Usage for verification: ./mach lint -l perfdocs + Usage for generation: Not Implemented + + Currently, doc generation is not implemented - only validation. + + For validation, see the Verifier class for a description of how + it works. + + The run will fail if the valid result from validate_tree is not + False, implying some warning/problem was logged. + + :param dict config: The configuration given by mozlint. + :param StructuredLogger logger: The StructuredLogger instance to be used to + output the linting warnings/errors. + :param list paths: The paths that are being tested. Used to filter + out errors from files outside of these paths. + :param bool verify: If true, the verification will be performed. + :param bool generate: If true, the docs will be generated. + ''' + from perfdocs.logger import PerfDocLogger + + top_dir = os.environ.get('WORKSPACE', None) + if not top_dir: + floc = os.path.abspath(__file__) + top_dir = floc.split('tools')[0] + + PerfDocLogger.LOGGER = logger + # Convert all the paths to relative ones + rel_paths = [re.sub(".*testing", "testing", path) for path in paths] + PerfDocLogger.PATHS = rel_paths + + # TODO: Expand search to entire tree rather than just the testing directory + testing_dir = os.path.join(top_dir, 'testing') + if not os.path.exists(testing_dir): + raise Exception("Cannot locate testing directory at %s" % testing_dir) + + # Run either the verifier or generator + if generate: + raise NotImplementedError + if verify: + from perfdocs.verifier import Verifier + + verifier = Verifier(testing_dir, top_dir) + verifier.validate_tree() diff --git a/tools/lint/perfdocs/requirements.txt b/tools/lint/perfdocs/requirements.txt new file mode 100644 index 000000000000..d3ef59461748 --- /dev/null +++ b/tools/lint/perfdocs/requirements.txt @@ -0,0 +1,8 @@ +jsonschema==3.1.1 --hash=sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631 +importlib-metadata==0.23 --hash=sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af +attrs==17.4.0 --hash=sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450 +setuptools==41.6.0 --hash=sha256:3e8e8505e563631e7cb110d9ad82d135ee866b8146d5efe06e42be07a72db20a +pyrsistent==0.15.5 --hash=sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778 +zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335 +six==1.13.0 --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd +more-itertools==7.2.0 --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4 diff --git a/tools/lint/perfdocs/utils.py b/tools/lint/perfdocs/utils.py new file mode 100644 index 000000000000..a7db7b753a62 --- /dev/null +++ b/tools/lint/perfdocs/utils.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import yaml +from perfdocs.logger import PerfDocLogger + +logger = PerfDocLogger() + + +def read_file(path): + '''Opens a file and returns its contents. + + :param str path: Path to the file. + :return list: List containing the lines in the file. + ''' + with open(path, 'r') as f: + return f.readlines() + + +def read_yaml(yaml_path): + ''' + Opens a YAML file and returns the contents. + + :param str yaml_path: Path to the YAML to open. + :return dict: Dictionary containing the YAML content. + ''' + contents = {} + try: + with open(yaml_path, 'r') as f: + contents = yaml.safe_load(f) + except Exception as e: + logger.warning( + "Error opening file {}: {}".format(yaml_path, str(e)), yaml_path + ) + + return contents diff --git a/tools/lint/perfdocs/verifier.py b/tools/lint/perfdocs/verifier.py new file mode 100644 index 000000000000..0d4357ecfe86 --- /dev/null +++ b/tools/lint/perfdocs/verifier.py @@ -0,0 +1,307 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import + +import jsonschema +import os +import re + +from perfdocs.logger import PerfDocLogger +from perfdocs.utils import read_file, read_yaml +from perfdocs.gatherer import Gatherer + +logger = PerfDocLogger() + +''' +Schema for the config.yml file. +Expecting a YAML file with a format such as this: + +name: raptor +manifest testing/raptor/raptor/raptor.ini +suites: + desktop: + description: "Desktop tests." + tests: + raptor-tp6: "Raptor TP6 tests." + mobile: + description: "Mobile tests" + benchmarks: + description: "Benchmark tests." + tests: + wasm: "All wasm tests." + +''' +CONFIG_SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "manifest": {"type": "string"}, + "suites": { + "type": "object", + "properties": { + "suite_name": { + "type": "object", + "properties": { + "tests": { + "type": "object", + "properties": { + "test_name": {"type": "string"}, + } + }, + "description": {"type": "string"}, + }, + "required": [ + "description" + ] + } + } + } + }, + "required": [ + "name", + "manifest", + "suites" + ] +} + + +class Verifier(object): + ''' + Verifier is used for validating the perfdocs folders/tree. In the future, + the generator will make use of this class to obtain a validated set of + descriptions that can be used to build up a document. + ''' + + def __init__(self, root_dir, workspace_dir): + ''' + Initialize the Verifier. + + :param str root_dir: Path to the 'testing' directory. + :param str workspace_dir: Path to the top-level checkout directory. + ''' + self.workspace_dir = workspace_dir + self._gatherer = Gatherer(root_dir, workspace_dir) + + def validate_descriptions(self, framework_info): + ''' + Cross-validate the tests found in the manifests and the YAML + test definitions. This function doesn't return a valid flag. Instead, + the StructDocLogger.VALIDATION_LOG is used to determine validity. + + The validation proceeds as follows: + 1. Check that all tests/suites in the YAML exist in the manifests. + - At the same time, build a list of global descriptions which + define descriptions for groupings of tests. + 2. Check that all tests/suites found in the manifests exist in the YAML. + - For missing tests, check if a global description for them exists. + + As the validation is completed, errors are output into the validation log + for any issues that are found. + + :param dict framework_info: Contains information about the framework. See + `Gatherer.get_test_list` for information about its structure. + ''' + yaml_content = framework_info['yml_content'] + + # Check for any bad test/suite names in the yaml config file + global_descriptions = {} + for suite, ytests in yaml_content['suites'].items(): + # Find the suite, then check against the tests within it + if framework_info["test_list"].get(suite): + global_descriptions[suite] = [] + if not ytests.get("tests"): + # It's possible a suite entry has no tests + continue + + # Suite found - now check if any tests in YAML + # definitions doesn't exist + ytests = ytests['tests'] + for mnf_pth in ytests: + foundtest = False + for t in framework_info["test_list"][suite]: + tb = os.path.basename(t) + tb = re.sub("\..*", "", tb) + if mnf_pth == tb: + # Found an exact match for the mnf_pth + foundtest = True + break + if mnf_pth in tb: + # Found a 'fuzzy' match for the mnf_pth + # i.e. 'wasm' could exist for all raptor wasm tests + global_descriptions[suite].append(mnf_pth) + foundtest = True + break + if not foundtest: + logger.warning( + "Could not find an existing test for {} - bad test name?".format( + mnf_pth + ), + framework_info["yml_path"] + ) + else: + logger.warning( + "Could not find an existing suite for {} - bad suite name?".format(suite), + framework_info["yml_path"] + ) + + # Check for any missing tests/suites + for suite, manifest_paths in framework_info["test_list"].items(): + if not yaml_content["suites"].get(suite): + # Description doesn't exist for the suite + logger.warning( + "Missing suite description for {}".format(suite), + yaml_content['manifest'] + ) + continue + + # If only a description is provided for the suite, assume + # that this is a suite-wide description and don't check for + # it's tests + stests = yaml_content['suites'][suite].get('tests', None) + if not stests: + continue + + tests_found = 0 + missing_tests = [] + test_to_manifest = {} + for mnf_pth in manifest_paths: + tb = os.path.basename(mnf_pth) + tb = re.sub("\..*", "", tb) + if stests.get(tb) or stests.get(mnf_pth): + # Test description exists, continue with the next test + tests_found += 1 + continue + test_to_manifest[tb] = mnf_pth + missing_tests.append(tb) + + # Check if global test descriptions exist (i.e. + # ones that cover all of tp6) for the missing tests + new_mtests = [] + for mt in missing_tests: + found = False + for mnf_pth in global_descriptions[suite]: + if mnf_pth in mt: + # Global test exists for this missing test + found = True + break + if not found: + new_mtests.append(mt) + + if len(new_mtests): + # Output an error for each manifest with a missing + # test description + for mnf_pth in new_mtests: + logger.warning( + "Could not find a test description for {}".format(mnf_pth), + test_to_manifest[mnf_pth] + ) + continue + + def validate_yaml(self, yaml_path): + ''' + Validate that the YAML file has all the fields that are + required and parse the descriptions into strings in case + some are give as relative file paths. + + :param str yaml_path: Path to the YAML to validate. + :return bool: True/False => Passed/Failed Validation + ''' + def _get_description(desc): + ''' + Recompute the description in case it's a file. + ''' + desc_path = os.path.join(self.workspace_dir, desc) + if os.path.exists(desc_path): + with open(desc_path, 'r') as f: + desc = f.readlines() + return desc + + def _parse_descriptions(content): + for suite, sinfo in content.items(): + desc = sinfo['description'] + sinfo['description'] = _get_description(desc) + + # It's possible that the suite has no tests and + # only a description. If they exist, then parse them. + if 'tests' in sinfo: + for test, desc in sinfo['tests'].items(): + sinfo['tests'][test] = _get_description(desc) + + valid = False + yaml_content = read_yaml(yaml_path) + + try: + jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA) + _parse_descriptions(yaml_content['suites']) + valid = True + except Exception as e: + logger.warning( + "YAML ValidationError: {}".format(str(e)), yaml_path + ) + + return valid + + def validate_rst_content(self, rst_path): + ''' + Validate that the index file given has a {documentation} entry + so that the documentation can be inserted there. + + :param str rst_path: Path to the RST file. + :return bool: True/False => Passed/Failed Validation + ''' + rst_content = read_file(rst_path) + + # Check for a {documentation} entry in some line, + # if we can't find one, then the validation fails. + valid = False + docs_match = re.compile('.*{documentation}.*') + for line in rst_content: + if docs_match.search(line): + valid = True + break + if not valid: + logger.warning( + "Cannot find a '{documentation}' entry in the given index file", + rst_path + ) + + return valid + + def _check_framework_descriptions(self, item): + ''' + Helper method for validating descriptions + ''' + framework_info = self._gatherer.get_test_list(item) + self.validate_descriptions(framework_info) + + def validate_tree(self): + ''' + Validate the `perfdocs` directory that was found. + Returns True if it is good, false otherwise. + + :return bool: True/False => Passed/Failed Validation + ''' + found_good = 0 + + # For each framework, check their files and validate descriptions + for matched in self._gatherer.perfdocs_tree: + # Get the paths to the YAML and RST for this framework + matched_yml = os.path.join(matched['path'], matched['yml']) + matched_rst = os.path.join(matched['path'], matched['rst']) + + _valid_files = { + "yml": self.validate_yaml(matched_yml), + "rst": self.validate_rst_content(matched_rst) + } + + if not all(_valid_files.values()): + # Don't check the descriptions if the YAML or RST is bad + logger.log("Bad perfdocs directory found in {}".format(matched['path'])) + continue + found_good += 1 + + self._check_framework_descriptions(matched) + + if not found_good: + raise Exception("No valid perfdocs directories found")