Bug 1587080 - Part 1: Add performance documentation verification code. r=ahal,perftest-reviewers,alexandru.irimovici,octavian_negru

This patch adds the performance documentation (perfdocs) verification code under `tools/lint/perfdocs`. This tool currently validates `perfdocs` folders found within the `testing` folder to ensure all performance tests have documentation (it only does this for raptor at the moment). See `tools/lint/docs/perfdocs.rst` for more details.

Differential Revision: https://phabricator.services.mozilla.com/D53647

--HG--
extra : moz-landing-system : lando

tools/lint/perfdocs/__init__.py

@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+import os
+
+from perfdocs import perfdocs
+from mozlint.util import pip
+
+here = os.path.abspath(os.path.dirname(__file__))
+PERFDOCS_REQUIREMENTS_PATH = os.path.join(here, 'requirements.txt')
+
+
+def setup(root, **lintargs):
+    if not pip.reinstall_program(PERFDOCS_REQUIREMENTS_PATH):
+        print("Cannot install requirements.")
+        return 1
+
+
+def lint(paths, config, logger, fix=None, **lintargs):
+    return perfdocs.run_perfdocs(
+        config, logger=logger, paths=paths, verify=True
+    )

tools/lint/perfdocs/framework_gatherers.py

@@ -0,0 +1,107 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import os
+import re
+
+from perfdocs.utils import read_yaml
+from manifestparser import TestManifest
+
+'''
+This file is for framework specific gatherers since manifests
+might be parsed differently in each of them. The gatherers
+must implement the FrameworkGatherer class.
+'''
+
+
+class FrameworkGatherer(object):
+    '''
+    Abstract class for framework gatherers.
+    '''
+
+    def __init__(self, yaml_path, workspace_dir):
+        '''
+        Generic initialization for a framework gatherer.
+        '''
+        self.workspace_dir = workspace_dir
+        self._yaml_path = yaml_path
+        self._suite_list = {}
+        self._manifest_path = ''
+        self._manifest = None
+
+    def get_manifest_path(self):
+        '''
+        Returns the path to the manifest based on the
+        manifest entry in the frameworks YAML configuration
+        file.
+
+        :return str: Path to the manifest.
+        '''
+        if self._manifest_path:
+            return self._manifest_path
+
+        yaml_content = read_yaml(self._yaml_path)
+        self._manifest_path = os.path.join(
+            self.workspace_dir, yaml_content["manifest"]
+        )
+        return self._manifest_path
+
+    def get_suite_list(self):
+        '''
+        Each framework gatherer must return a dictionary with
+        the following structure. Note that the test names must
+        be relative paths so that issues can be correctly issued
+        by the reviewbot.
+
+        :return dict: A dictionary with the following structure: {
+                "suite_name": [
+                    'testing/raptor/test1',
+                    'testing/raptor/test2'
+                ]
+            }
+        '''
+        raise NotImplementedError
+
+
+class RaptorGatherer(FrameworkGatherer):
+    '''
+    Gatherer for the Raptor framework.
+    '''
+
+    def get_suite_list(self):
+        '''
+        Returns a dictionary containing a mapping from suites
+        to the tests they contain.
+
+        :return dict: A dictionary with the following structure: {
+                "suite_name": [
+                    'testing/raptor/test1',
+                    'testing/raptor/test2'
+                ]
+            }
+        '''
+        if self._suite_list:
+            return self._suite_list
+
+        manifest_path = self.get_manifest_path()
+
+        # Get the tests from the manifest
+        test_manifest = TestManifest([manifest_path], strict=False)
+        test_list = test_manifest.active_tests(exists=False, disabled=False)
+
+        # Parse the tests into the expected dictionary
+        for test in test_list:
+            # Get the top-level suite
+            s = os.path.basename(test["here"])
+            if s not in self._suite_list:
+                self._suite_list[s] = []
+
+            # Get the individual test
+            fpath = re.sub(".*testing", "testing", test['manifest'])
+
+            if fpath not in self._suite_list[s]:
+                self._suite_list[s].append(fpath)
+
+        return self._suite_list

tools/lint/perfdocs/gatherer.py

@@ -0,0 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import os
+import re
+
+from perfdocs.logger import PerfDocLogger
+from perfdocs.utils import read_yaml
+from perfdocs.framework_gatherers import RaptorGatherer
+
+logger = PerfDocLogger()
+
+# TODO: Implement decorator/searcher to find the classes.
+frameworks = {
+    "raptor": RaptorGatherer,
+}
+
+
+class Gatherer(object):
+    '''
+    Gatherer produces the tree of the perfdoc's entries found
+    and can obtain manifest-based test lists. Used by the Verifier.
+    '''
+
+    def __init__(self, root_dir, workspace_dir):
+        '''
+        Initialzie the Gatherer.
+
+        :param str root_dir: Path to the testing directory.
+        :param str workspace_dir: Path to the gecko checkout.
+        '''
+        self.root_dir = root_dir
+        self.workspace_dir = workspace_dir
+        self._perfdocs_tree = []
+        self._test_list = []
+
+    @property
+    def perfdocs_tree(self):
+        '''
+        Returns the perfdocs_tree, and computes it
+        if it doesn't exist.
+
+        :return dict: The perfdocs tree containing all
+            framework perfdoc entries. See `fetch_perfdocs_tree`
+            for information on the data strcture.
+        '''
+        if self._perfdocs_tree:
+            return self.perfdocs_tree
+        else:
+            self.fetch_perfdocs_tree()
+            return self._perfdocs_tree
+
+    def fetch_perfdocs_tree(self):
+        '''
+        Creates the perfdocs tree with the following structure:
+            [
+                {
+                    "path": Path to the perfdocs directory.
+                    "yml": Name of the configuration YAML file.
+                    "rst": Name of the RST file.
+                }, ...
+            ]
+
+        This method doesn't return anything. The result can be found in
+        the perfdocs_tree attribute.
+        '''
+        yml_match = re.compile('^config.y(a)?ml$')
+        rst_match = re.compile('^index.rst$')
+
+        for dirpath, dirname, files in os.walk(self.root_dir):
+            # Walk through the testing directory tree
+            if dirpath.endswith('/perfdocs'):
+                matched = {"path": dirpath, "yml": "", "rst": ""}
+                for file in files:
+                    # Add the yml/rst file to its key if re finds the searched file
+                    if re.search(yml_match, file):
+                        matched["yml"] = re.search(yml_match, file).string
+                    if re.search(rst_match, file):
+                        matched["rst"] = re.search(rst_match, file).string
+                    # Append to structdocs if all the searched files were found
+                    if all(matched.values()):
+                        self._perfdocs_tree.append(matched)
+
+        logger.log("Found {} perfdocs directories in {}"
+                   .format(len(self._perfdocs_tree), self.root_dir))
+
+    def get_test_list(self, sdt_entry):
+        '''
+        Use a perfdocs_tree entry to find the test list for
+        the framework that was found.
+
+        :return: A framework info dictionary with fields: {
+            'yml_path': Path to YAML,
+            'yml_content': Content of YAML,
+            'name': Name of framework,
+            'test_list': Test list found for the framework
+        }
+        '''
+
+        # If it was computed before, return it
+        yaml_path = os.path.join(sdt_entry["path"], sdt_entry['yml'])
+        for entry in self._test_list:
+            if entry['yml_path'] == yaml_path:
+                return entry
+
+        # Set up framework entry with meta data
+        yaml_content = read_yaml(yaml_path)
+        framework = {
+            'yml_content': yaml_content,
+            'yml_path': yaml_path,
+            'name': yaml_content["name"],
+        }
+
+        # Get and then store the frameworks tests
+        framework_gatherer = frameworks[framework["name"]](
+            framework["yml_path"],
+            self.workspace_dir
+        )
+        framework["test_list"] = framework_gatherer.get_suite_list()
+
+        self._test_list.append(framework)
+        return framework

tools/lint/perfdocs/logger.py

@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import re
+
+
+class PerfDocLogger(object):
+    '''
+    Logger for the PerfDoc tooling. Handles the warnings by outputting
+    them into through the StructuredLogger provided by lint.
+    '''
+    PATHS = []
+    LOGGER = None
+
+    def __init__(self):
+        '''Initializes the PerfDocLogger.'''
+
+        # Set up class attributes for all logger instances
+        if not PerfDocLogger.LOGGER:
+            raise Exception(
+                "Missing linting LOGGER instance for PerfDocLogger initialization"
+            )
+        if not PerfDocLogger.PATHS:
+            raise Exception(
+                "Missing PATHS for PerfDocLogger initialization"
+            )
+        self.logger = PerfDocLogger.LOGGER
+
+    def log(self, msg):
+        '''
+        Log a message.
+
+        :param str msg: Message to log.
+        '''
+        self.logger.info(msg)
+
+    def warning(self, msg, files):
+        '''
+        Logs a validation warning message. The warning message is
+        used as the error message that is output in the reviewbot.
+
+        :param str msg: Message to log, it's also used as the error message
+            for the issue that is output by the reviewbot.
+        :param list/str files: The file(s) that this warning is about.
+        '''
+        if type(files) != list:
+            files = [files]
+
+        # Add a reviewbot error for each file that is given
+        for file in files:
+            # Get a relative path (reviewbot can't handle absolute paths)
+            # TODO: Expand to outside of the testing directory
+            fpath = re.sub(".*testing", "testing", file)
+
+            # Filter out any issues that do not relate to the paths
+            # that are being linted
+            for path in PerfDocLogger.PATHS:
+                if path not in file:
+                    continue
+
+                # Output error entry
+                self.logger.lint_error(
+                    message=msg,
+                    lineno=0,
+                    column=None,
+                    path=fpath,
+                    linter='perfdocs',
+                    rule="Flawless performance docs."
+                )
+
+                break

tools/lint/perfdocs/perfdocs.py

@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function
+
+import os
+import re
+
+
+def run_perfdocs(config, logger=None, paths=None, verify=True, generate=False):
+    '''
+    Build up performance testing documentation dynamically by combining
+    text data from YAML files that reside in `perfdoc` folders
+    across the `testing` directory. Each directory is expected to have
+    an `index.rst` file along with `config.yml` YAMLs defining what needs
+    to be added to the documentation.
+
+    The YAML must also define the name of the "framework" that should be
+    used in the main index.rst for the performance testing documentation.
+
+    The testing documentation list will be ordered alphabetically once
+    it's produced (to avoid unwanted shifts because of unordered dicts
+    and path searching).
+
+    Note that the suite name headings will be given the H4 (---) style so it
+    is suggested that you use H3 (===) style as the heading for your
+    test section. H5 will be used be used for individual tests within each
+    suite.
+
+    Usage for verification: ./mach lint -l perfdocs
+    Usage for generation: Not Implemented
+
+    Currently, doc generation is not implemented - only validation.
+
+    For validation, see the Verifier class for a description of how
+    it works.
+
+    The run will fail if the valid result from validate_tree is not
+    False, implying some warning/problem was logged.
+
+    :param dict config: The configuration given by mozlint.
+    :param StructuredLogger logger: The StructuredLogger instance to be used to
+        output the linting warnings/errors.
+    :param list paths: The paths that are being tested. Used to filter
+        out errors from files outside of these paths.
+    :param bool verify: If true, the verification will be performed.
+    :param bool generate: If true, the docs will be generated.
+    '''
+    from perfdocs.logger import PerfDocLogger
+
+    top_dir = os.environ.get('WORKSPACE', None)
+    if not top_dir:
+        floc = os.path.abspath(__file__)
+        top_dir = floc.split('tools')[0]
+
+    PerfDocLogger.LOGGER = logger
+    # Convert all the paths to relative ones
+    rel_paths = [re.sub(".*testing", "testing", path) for path in paths]
+    PerfDocLogger.PATHS = rel_paths
+
+    # TODO: Expand search to entire tree rather than just the testing directory
+    testing_dir = os.path.join(top_dir, 'testing')
+    if not os.path.exists(testing_dir):
+        raise Exception("Cannot locate testing directory at %s" % testing_dir)
+
+    # Run either the verifier or generator
+    if generate:
+        raise NotImplementedError
+    if verify:
+        from perfdocs.verifier import Verifier
+
+        verifier = Verifier(testing_dir, top_dir)
+        verifier.validate_tree()

tools/lint/perfdocs/requirements.txt

@@ -0,0 +1,8 @@
+jsonschema==3.1.1 --hash=sha256:94c0a13b4a0616458b42529091624e66700a17f847453e52279e35509a5b7631
+importlib-metadata==0.23 --hash=sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af
+attrs==17.4.0 --hash=sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450
+setuptools==41.6.0 --hash=sha256:3e8e8505e563631e7cb110d9ad82d135ee866b8146d5efe06e42be07a72db20a
+pyrsistent==0.15.5 --hash=sha256:eb6545dbeb1aa69ab1fb4809bfbf5a8705e44d92ef8fc7c2361682a47c46c778 
+zipp==0.6.0 --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335
+six==1.13.0 --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd
+more-itertools==7.2.0 --hash=sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4

tools/lint/perfdocs/utils.py

@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import yaml
+from perfdocs.logger import PerfDocLogger
+
+logger = PerfDocLogger()
+
+
+def read_file(path):
+    '''Opens a file and returns its contents.
+
+    :param str path: Path to the file.
+    :return list: List containing the lines in the file.
+    '''
+    with open(path, 'r') as f:
+        return f.readlines()
+
+
+def read_yaml(yaml_path):
+    '''
+    Opens a YAML file and returns the contents.
+
+    :param str yaml_path: Path to the YAML to open.
+    :return dict: Dictionary containing the YAML content.
+    '''
+    contents = {}
+    try:
+        with open(yaml_path, 'r') as f:
+            contents = yaml.safe_load(f)
+    except Exception as e:
+        logger.warning(
+            "Error opening file {}: {}".format(yaml_path, str(e)), yaml_path
+        )
+
+    return contents

tools/lint/perfdocs/verifier.py

@@ -0,0 +1,307 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+
+import jsonschema
+import os
+import re
+
+from perfdocs.logger import PerfDocLogger
+from perfdocs.utils import read_file, read_yaml
+from perfdocs.gatherer import Gatherer
+
+logger = PerfDocLogger()
+
+'''
+Schema for the config.yml file.
+Expecting a YAML file with a format such as this:
+
+name: raptor
+manifest testing/raptor/raptor/raptor.ini
+suites:
+    desktop:
+        description: "Desktop tests."
+        tests:
+            raptor-tp6: "Raptor TP6 tests."
+    mobile:
+        description: "Mobile tests"
+    benchmarks:
+        description: "Benchmark tests."
+        tests:
+            wasm: "All wasm tests."
+
+'''
+CONFIG_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "manifest": {"type": "string"},
+        "suites": {
+            "type": "object",
+            "properties": {
+                "suite_name": {
+                    "type": "object",
+                    "properties": {
+                        "tests": {
+                            "type": "object",
+                            "properties": {
+                                "test_name": {"type": "string"},
+                            }
+                        },
+                        "description": {"type": "string"},
+                    },
+                    "required": [
+                        "description"
+                    ]
+                }
+            }
+        }
+    },
+    "required": [
+        "name",
+        "manifest",
+        "suites"
+    ]
+}
+
+
+class Verifier(object):
+    '''
+    Verifier is used for validating the perfdocs folders/tree. In the future,
+    the generator will make use of this class to obtain a validated set of
+    descriptions that can be used to build up a document.
+    '''
+
+    def __init__(self, root_dir, workspace_dir):
+        '''
+        Initialize the Verifier.
+
+        :param str root_dir: Path to the 'testing' directory.
+        :param str workspace_dir: Path to the top-level checkout directory.
+        '''
+        self.workspace_dir = workspace_dir
+        self._gatherer = Gatherer(root_dir, workspace_dir)
+
+    def validate_descriptions(self, framework_info):
+        '''
+        Cross-validate the tests found in the manifests and the YAML
+        test definitions. This function doesn't return a valid flag. Instead,
+        the StructDocLogger.VALIDATION_LOG is used to determine validity.
+
+        The validation proceeds as follows:
+            1. Check that all tests/suites in the YAML exist in the manifests.
+                - At the same time, build a list of global descriptions which
+                   define descriptions for groupings of tests.
+            2. Check that all tests/suites found in the manifests exist in the YAML.
+                - For missing tests, check if a global description for them exists.
+
+        As the validation is completed, errors are output into the validation log
+        for any issues that are found.
+
+        :param dict framework_info: Contains information about the framework. See
+            `Gatherer.get_test_list` for information about its structure.
+        '''
+        yaml_content = framework_info['yml_content']
+
+        # Check for any bad test/suite names in the yaml config file
+        global_descriptions = {}
+        for suite, ytests in yaml_content['suites'].items():
+            # Find the suite, then check against the tests within it
+            if framework_info["test_list"].get(suite):
+                global_descriptions[suite] = []
+                if not ytests.get("tests"):
+                    # It's possible a suite entry has no tests
+                    continue
+
+                # Suite found - now check if any tests in YAML
+                # definitions doesn't exist
+                ytests = ytests['tests']
+                for mnf_pth in ytests:
+                    foundtest = False
+                    for t in framework_info["test_list"][suite]:
+                        tb = os.path.basename(t)
+                        tb = re.sub("\..*", "", tb)
+                        if mnf_pth == tb:
+                            # Found an exact match for the mnf_pth
+                            foundtest = True
+                            break
+                        if mnf_pth in tb:
+                            # Found a 'fuzzy' match for the mnf_pth
+                            # i.e. 'wasm' could exist for all raptor wasm tests
+                            global_descriptions[suite].append(mnf_pth)
+                            foundtest = True
+                            break
+                    if not foundtest:
+                        logger.warning(
+                            "Could not find an existing test for {} - bad test name?".format(
+                                mnf_pth
+                            ),
+                            framework_info["yml_path"]
+                        )
+            else:
+                logger.warning(
+                    "Could not find an existing suite for {} - bad suite name?".format(suite),
+                    framework_info["yml_path"]
+                )
+
+        # Check for any missing tests/suites
+        for suite, manifest_paths in framework_info["test_list"].items():
+            if not yaml_content["suites"].get(suite):
+                # Description doesn't exist for the suite
+                logger.warning(
+                    "Missing suite description for {}".format(suite),
+                    yaml_content['manifest']
+                )
+                continue
+
+            # If only a description is provided for the suite, assume
+            # that this is a suite-wide description and don't check for
+            # it's tests
+            stests = yaml_content['suites'][suite].get('tests', None)
+            if not stests:
+                continue
+
+            tests_found = 0
+            missing_tests = []
+            test_to_manifest = {}
+            for mnf_pth in manifest_paths:
+                tb = os.path.basename(mnf_pth)
+                tb = re.sub("\..*", "", tb)
+                if stests.get(tb) or stests.get(mnf_pth):
+                    # Test description exists, continue with the next test
+                    tests_found += 1
+                    continue
+                test_to_manifest[tb] = mnf_pth
+                missing_tests.append(tb)
+
+            # Check if global test descriptions exist (i.e.
+            # ones that cover all of tp6) for the missing tests
+            new_mtests = []
+            for mt in missing_tests:
+                found = False
+                for mnf_pth in global_descriptions[suite]:
+                    if mnf_pth in mt:
+                        # Global test exists for this missing test
+                        found = True
+                        break
+                if not found:
+                    new_mtests.append(mt)
+
+            if len(new_mtests):
+                # Output an error for each manifest with a missing
+                # test description
+                for mnf_pth in new_mtests:
+                    logger.warning(
+                        "Could not find a test description for {}".format(mnf_pth),
+                        test_to_manifest[mnf_pth]
+                    )
+                continue
+
+    def validate_yaml(self, yaml_path):
+        '''
+        Validate that the YAML file has all the fields that are
+        required and parse the descriptions into strings in case
+        some are give as relative file paths.
+
+        :param str yaml_path: Path to the YAML to validate.
+        :return bool: True/False => Passed/Failed Validation
+        '''
+        def _get_description(desc):
+            '''
+            Recompute the description in case it's a file.
+            '''
+            desc_path = os.path.join(self.workspace_dir, desc)
+            if os.path.exists(desc_path):
+                with open(desc_path, 'r') as f:
+                    desc = f.readlines()
+            return desc
+
+        def _parse_descriptions(content):
+            for suite, sinfo in content.items():
+                desc = sinfo['description']
+                sinfo['description'] = _get_description(desc)
+
+                # It's possible that the suite has no tests and
+                # only a description. If they exist, then parse them.
+                if 'tests' in sinfo:
+                    for test, desc in sinfo['tests'].items():
+                        sinfo['tests'][test] = _get_description(desc)
+
+        valid = False
+        yaml_content = read_yaml(yaml_path)
+
+        try:
+            jsonschema.validate(instance=yaml_content, schema=CONFIG_SCHEMA)
+            _parse_descriptions(yaml_content['suites'])
+            valid = True
+        except Exception as e:
+            logger.warning(
+                "YAML ValidationError: {}".format(str(e)), yaml_path
+            )
+
+        return valid
+
+    def validate_rst_content(self, rst_path):
+        '''
+        Validate that the index file given has a {documentation} entry
+        so that the documentation can be inserted there.
+
+        :param str rst_path: Path to the RST file.
+        :return bool: True/False => Passed/Failed Validation
+        '''
+        rst_content = read_file(rst_path)
+
+        # Check for a {documentation} entry in some line,
+        # if we can't find one, then the validation fails.
+        valid = False
+        docs_match = re.compile('.*{documentation}.*')
+        for line in rst_content:
+            if docs_match.search(line):
+                valid = True
+                break
+        if not valid:
+            logger.warning(
+                "Cannot find a '{documentation}' entry in the given index file",
+                rst_path
+            )
+
+        return valid
+
+    def _check_framework_descriptions(self, item):
+        '''
+        Helper method for validating descriptions
+        '''
+        framework_info = self._gatherer.get_test_list(item)
+        self.validate_descriptions(framework_info)
+
+    def validate_tree(self):
+        '''
+        Validate the `perfdocs` directory that was found.
+        Returns True if it is good, false otherwise.
+
+        :return bool: True/False => Passed/Failed Validation
+        '''
+        found_good = 0
+
+        # For each framework, check their files and validate descriptions
+        for matched in self._gatherer.perfdocs_tree:
+            # Get the paths to the YAML and RST for this framework
+            matched_yml = os.path.join(matched['path'], matched['yml'])
+            matched_rst = os.path.join(matched['path'], matched['rst'])
+
+            _valid_files = {
+                "yml": self.validate_yaml(matched_yml),
+                "rst": self.validate_rst_content(matched_rst)
+            }
+
+            if not all(_valid_files.values()):
+                # Don't check the descriptions if the YAML or RST is bad
+                logger.log("Bad perfdocs directory found in {}".format(matched['path']))
+                continue
+            found_good += 1
+
+            self._check_framework_descriptions(matched)
+
+        if not found_good:
+            raise Exception("No valid perfdocs directories found")