Bug 1184405 - Add annotations for tags, file patterns, and test flavors to moz.build to specify tests potentially impacted by source files. r=gps

--HG--
extra : commitid : JbsAq6sNQ9f

python/mozbuild/mozbuild/frontend/context.py

@@ -35,9 +35,10 @@ from mozbuild.util import (
     TypedList,
     TypedNamedTuple,
 )
+
+from ..testing import all_test_flavors
 import mozpack.path as mozpath
 from types import FunctionType
-from UserString import UserString
 
 import itertools
 
@@ -528,6 +529,14 @@ WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest",
                                           [("manifest_path", unicode),
                                            ("test_root", unicode)])
 
+OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList)
+OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
+                                  StrictOrderingOnAppendList)
+OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList)
+DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
+                                                ('tags', OrderedStringList),
+                                                ('flavors', OrderedTestFlavorList))
+
 class Files(SubContext):
     """Metadata attached to files.
 
@@ -596,17 +605,79 @@ class Files(SubContext):
 
             See :ref:`mozbuild_files_metadata_finalizing` for more info.
             """, None),
+        'IMPACTED_TESTS': (DependentTestsEntry, list,
+            """File patterns, tags, and flavors for tests relevant to these files.
+
+            Maps source files to the tests potentially impacted by those files.
+            Tests can be specified by file pattern, tag, or flavor.
+
+            For example:
+
+            with Files('runtests.py'):
+               IMPACTED_TESTS.files += [
+                   '**',
+               ]
+
+            in testing/mochitest/moz.build will suggest that any of the tests
+            under testing/mochitest may be impacted by a change to runtests.py.
+
+            File patterns may be made relative to the topsrcdir with a leading
+            '/', so
+
+            with Files('httpd.js'):
+               IMPACTED_TESTS.files += [
+                   '/testing/mochitest/tests/Harness_sanity/**',
+               ]
+
+            in netwerk/test/httpserver/moz.build will suggest that any change to httpd.js
+            will be relevant to the mochitest sanity tests.
+
+            Tags and flavors are sorted string lists (flavors are limited to valid
+            values).
+
+            For example:
+
+            with Files('toolkit/devtools/*'):
+                IMPACTED_TESTS.tags += [
+                    'devtools',
+                ]
+
+            in the root moz.build would suggest that any test tagged 'devtools' would
+            potentially be impacted by a change to a file under toolkit/devtools, and
+
+            with Files('dom/base/nsGlobalWindow.cpp'):
+                IMPACTED_TESTS.flavors += [
+                    'mochitest',
+                ]
+
+            Would suggest that nsGlobalWindow.cpp is potentially relevant to
+            any plain mochitest.
+            """, None),
     }
 
     def __init__(self, parent, pattern=None):
         super(Files, self).__init__(parent)
         self.pattern = pattern
         self.finalized = set()
+        self.test_files = set()
+        self.test_tags = set()
+        self.test_flavors = set()
 
     def __iadd__(self, other):
         assert isinstance(other, Files)
 
+        self.test_files |= other.test_files
+        self.test_tags |= other.test_tags
+        self.test_flavors |= other.test_flavors
+
         for k, v in other.items():
+            if k == 'IMPACTED_TESTS':
+                self.test_files |= set(mozpath.relpath(e.full_path, e.context.config.topsrcdir)
+                                       for e in v.files)
+                self.test_tags |= set(v.tags)
+                self.test_flavors |= set(v.flavors)
+                continue
+
             # Ignore updates to finalized flags.
             if k in self.finalized:
                 continue

python/mozbuild/mozbuild/frontend/data.py

@@ -28,6 +28,10 @@ from ..util import (
     group_unified_files,
 )
 
+from ..testing import (
+    all_test_flavors,
+)
+
 
 class TreeMetadata(object):
     """Base class for all data being captured."""
@@ -630,6 +634,8 @@ class TestManifest(ContextDerived):
             install_prefix=None, relpath=None, dupe_manifest=False):
         ContextDerived.__init__(self, context)
 
+        assert flavor in all_test_flavors()
+
         self.path = path
         self.directory = mozpath.dirname(path)
         self.manifest = manifest

python/mozbuild/mozbuild/frontend/emitter.py

@@ -80,9 +80,14 @@ from .data import (
 
 from .reader import SandboxValidationError
 
+from ..testing import (
+    TEST_MANIFESTS,
+    REFTEST_FLAVORS,
+    WEB_PATFORM_TESTS_FLAVORS,
+)
+
 from .context import (
     Context,
-    AbsolutePath,
     SourcePath,
     ObjDirPath,
     Path,
@@ -980,51 +985,21 @@ class TreeMetadataEmitter(LoggingMixin):
                     else 'USE_LIBS'))
 
     def _process_test_manifests(self, context):
-        # While there are multiple test manifests, the behavior is very similar
-        # across them. We enforce this by having common handling of all
-        # manifests and outputting a single class type with the differences
-        # described inside the instance.
-        #
-        # Keys are variable prefixes and values are tuples describing how these
-        # manifests should be handled:
-        #
-        #    (flavor, install_prefix, package_tests)
-        #
-        # flavor identifies the flavor of this test.
-        # install_prefix is the path prefix of where to install the files in
-        #     the tests directory.
-        # package_tests indicates whether to package test files into the test
-        #     package; suites that compile the test files should not install
-        #     them into the test package.
-        #
-        test_manifests = dict(
-            A11Y=('a11y', 'testing/mochitest', 'a11y', True),
-            BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
-            ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
-            JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
-            JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
-            METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
-            MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
-            MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
-            MOCHITEST_WEBAPPRT_CONTENT=('webapprt-content', 'testing/mochitest', 'webapprtContent', True),
-            MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
-            WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
-            XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
-        )
 
-        for prefix, info in test_manifests.items():
+        for prefix, info in TEST_MANIFESTS.items():
             for path in context.get('%s_MANIFESTS' % prefix, []):
                 for obj in self._process_test_manifest(context, info, path):
                     yield obj
 
-        for flavor in ('crashtest', 'reftest'):
+        for flavor in REFTEST_FLAVORS:
             for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
                 for obj in self._process_reftest_manifest(context, flavor, path):
                     yield obj
 
-        for path in context.get("WEB_PLATFORM_TESTS_MANIFESTS", []):
-            for obj in self._process_web_platform_tests_manifest(context, path):
-                yield obj
+        for flavor in WEB_PATFORM_TESTS_FLAVORS:
+            for path in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
+                for obj in self._process_web_platform_tests_manifest(context, path):
+                    yield obj
 
     def _process_test_manifest(self, context, info, manifest_path):
         flavor, install_root, install_subdir, package_tests = info
@@ -1290,6 +1265,7 @@ class TreeMetadataEmitter(LoggingMixin):
 
         # Some paths have a subconfigure, yet also have a moz.build. Those
         # shouldn't end up in self._external_paths.
-        self._external_paths -= { o.relobjdir }
+        if o.objdir:
+            self._external_paths -= { o.relobjdir }
 
         yield o

python/mozbuild/mozbuild/frontend/reader.py

@@ -25,7 +25,6 @@ import os
 import sys
 import textwrap
 import time
-import tokenize
 import traceback
 import types
 
@@ -41,6 +40,12 @@ from mozbuild.util import (
     ReadOnlyDefaultDict,
 )
 
+from mozbuild.testing import (
+    TEST_MANIFESTS,
+    REFTEST_FLAVORS,
+    WEB_PATFORM_TESTS_FLAVORS,
+)
+
 from mozbuild.backend.configenvironment import ConfigEnvironment
 
 from mozpack.files import FileFinder
@@ -1328,6 +1333,7 @@ class BuildReader(object):
         paths, _ = self.read_relevant_mozbuilds(paths)
 
         r = {}
+        test_ctx_reader = TestContextReader(self.config)
 
         for path, ctxs in paths.items():
             flags = Files(Context())
@@ -1346,6 +1352,62 @@ class BuildReader(object):
                         ('*' in pattern and mozpath.match(relpath, pattern)):
                     flags += ctx
 
+            if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
+                flags += test_ctx_reader.test_defaults_for_path(path, ctxs)
+
             r[path] = flags
 
         return r
+
+
+class TestContextReader(object):
+    """Helper to extract test patterns defaults from moz.build files.
+
+    Given paths of interest and relevant contexts, populates a Files
+    object with patterns matching all tests mentioned in the given
+    contexts.
+    """
+    def __init__(self, config):
+        self.config = config
+
+        # This names the context keys that will end up emitting a test
+        # manifest.
+        self._test_manifest_contexts = set(
+            ['%s_MANIFESTS' % key for key in TEST_MANIFESTS] +
+            ['%s_MANIFESTS' % flavor.upper() for flavor in REFTEST_FLAVORS] +
+            ['%s_MANIFESTS' % flavor.upper().replace('-', '_') for flavor in WEB_PATFORM_TESTS_FLAVORS] +
+            # The emitter requires JAR_MANIFESTS in contexts if they exist on
+            # disk, so include them here.
+            ['JAR_MANIFESTS']
+        )
+
+
+    def test_defaults_for_path(self, path, ctxs):
+        # Using the emitter here creates a circular import (and crosses some
+        # abstraction boundaries), but it's a convenient way to get the build
+        # system's view of tests.
+        # Bug 1203266 tracks features that would allow improving this situation
+        # and removing this abuse of the TreeMetadataEmitter
+        from .emitter import TreeMetadataEmitter, TestManifest
+        emitter = TreeMetadataEmitter(self.config)
+
+        result_context = Files(Context())
+
+        for ctx in ctxs:
+            test_context = Context(VARIABLES, self.config)
+            test_context.main_path = ctx.main_path
+
+            # Clone just the keys that will result in test manifests.
+            manifest_keys = [key for key in ctx if key in self._test_manifest_contexts]
+            for key in manifest_keys:
+                test_context[key] = ctx[key]
+            for obj in emitter.emit_from_context(test_context):
+                if isinstance(obj, TestManifest):
+                    for t in obj.tests:
+                        if 'relpath' not in t:
+                            # wpt manifests do not generate relpaths (bug 1207678).
+                            t['relpath'] = mozpath.relpath(t['path'],
+                                                           self.config.topsrcdir)
+                        # Pull in the entire directory of tests.
+                        result_context.test_files.add(mozpath.dirname(t['relpath']) + '/**')
+        return result_context

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build

@@ -0,0 +1,5 @@
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini']
+
+EXTRA_JS_MODULES += [
+    'module.js',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini

@@ -0,0 +1 @@
+[test_default_mod.js]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build

@@ -0,0 +1,4 @@
+DIRS += [
+  'default',
+  'simple',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini

@@ -0,0 +1 @@
+[test_mod.js]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build

@@ -0,0 +1,22 @@
+with Files('src/*'):
+    IMPACTED_TESTS.files += [
+        'tests/test_general.html',
+    ]
+
+with Files('src/module.jsm'):
+    IMPACTED_TESTS.files += [
+        'browser/**.js',
+    ]
+
+with Files('base.cpp'):
+    IMPACTED_TESTS.files += [
+        '/default/tests/xpcshell/test_default_mod.js',
+        'tests/*',
+    ]
+
+
+MOCHITEST_MANIFESTS += ['tests/mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['browser/browser.ini']
+
+UNIFIED_SOURCES += ['base.cpp']
+DIRS += ['src']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build

@@ -0,0 +1,3 @@
+EXTRA_JS_MODULES += [
+  'module.jsm',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini

@@ -0,0 +1,2 @@
+[test_general.html]
+[test_specific.html]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build

@@ -0,0 +1 @@
+MOCHITEST_MANIFESTS += ['mochitest.ini']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build

@@ -0,0 +1,15 @@
+with Files('src/submodule/**'):
+     IMPACTED_TESTS.tags += [
+         'submodule',
+     ]
+
+with Files('src/bar.jsm'):
+    IMPACTED_TESTS.flavors += [
+        'browser-chrome',
+    ]
+    IMPACTED_TESTS.files += [
+        '**.js',
+    ]
+
+MOCHITEST_MANIFESTS += ['tests/mochitest.ini']
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini

@@ -0,0 +1,3 @@
+[test_simple.html]
+[test_specific.html]
+tags = submodule

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini

@@ -0,0 +1 @@
+[test_bar.js]

python/mozbuild/mozbuild/test/frontend/test_reader.py

@@ -29,6 +29,14 @@ data_path = mozpath.join(data_path, 'data')
 
 
 class TestBuildReader(unittest.TestCase):
+    def setUp(self):
+        self._old_env = dict(os.environ)
+        os.environ.pop('MOZ_OBJDIR', None)
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._old_env)
+
     def config(self, name, **kwargs):
         path = mozpath.join(data_path, name)
 
@@ -378,6 +386,77 @@ class TestBuildReader(unittest.TestCase):
         self.assertEqual(v['bug_component/final/subcomponent/bar']['BUG_COMPONENT'],
             BugzillaComponent('Another', 'Component'))
 
+    def test_file_test_deps(self):
+        reader = self.reader('files-test-metadata')
+
+        expected = {
+            'simple/src/module.jsm': set(['simple/tests/test_general.html',
+                                          'simple/browser/**.js']),
+            'simple/base.cpp': set(['simple/tests/*',
+                                    'default/tests/xpcshell/test_default_mod.js']),
+        }
+
+        v = reader.files_info([
+            'simple/src/module.jsm',
+            'simple/base.cpp',
+        ])
+
+        for path, pattern_set in expected.items():
+            self.assertEqual(v[path].test_files,
+                             expected[path])
+
+    def test_file_test_deps_default(self):
+        reader = self.reader('files-test-metadata')
+        v = reader.files_info([
+            'default/module.js',
+        ])
+
+        expected = {
+            'default/module.js': set(['default/tests/xpcshell/**']),
+        }
+
+        for path, pattern_set in expected.items():
+            self.assertEqual(v[path].test_files,
+                             expected[path])
+
+    def test_file_test_deps_tags(self):
+        reader = self.reader('files-test-metadata')
+        v = reader.files_info([
+            'tagged/src/bar.jsm',
+            'tagged/src/submodule/foo.js',
+        ])
+
+        expected_patterns = {
+            'tagged/src/submodule/foo.js': set([]),
+            'tagged/src/bar.jsm': set(['tagged/**.js']),
+        }
+
+        for path, pattern_set in expected_patterns.items():
+            self.assertEqual(v[path].test_files,
+                             expected_patterns[path])
+
+        expected_tags = {
+            'tagged/src/submodule/foo.js': set(['submodule']),
+            'tagged/src/bar.jsm': set([]),
+        }
+        for path, pattern_set in expected_tags.items():
+            self.assertEqual(v[path].test_tags,
+                             expected_tags[path])
+
+        expected_flavors = {
+            'tagged/src/bar.jsm': set(['browser-chrome']),
+            'tagged/src/submodule/foo.js': set([]),
+        }
+        for path, pattern_set in expected_flavors.items():
+            self.assertEqual(v[path].test_flavors,
+                             expected_flavors[path])
+
+    def test_invalid_flavor(self):
+        reader = self.reader('invalid-files-flavor')
+
+        with self.assertRaises(BuildReaderError):
+            reader.files_info(['foo.js'])
+
 
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/testing.py

@@ -225,3 +225,51 @@ class TestResolver(MozbuildObject):
                     honor_install_to_subdir=True)
             else:
                 yield test
+
+# These definitions provide a single source of truth for modules attempting
+# to get a view of all tests for a build. Used by the emitter to figure out
+# how to read/install manifests and by test dependency annotations in Files()
+# entries to enumerate test flavors.
+
+# While there are multiple test manifests, the behavior is very similar
+# across them. We enforce this by having common handling of all
+# manifests and outputting a single class type with the differences
+# described inside the instance.
+#
+# Keys are variable prefixes and values are tuples describing how these
+# manifests should be handled:
+#
+#    (flavor, install_prefix, package_tests)
+#
+# flavor identifies the flavor of this test.
+# install_prefix is the path prefix of where to install the files in
+#     the tests directory.
+# package_tests indicates whether to package test files into the test
+#     package; suites that compile the test files should not install
+#     them into the test package.
+#
+TEST_MANIFESTS = dict(
+    A11Y=('a11y', 'testing/mochitest', 'a11y', True),
+    BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
+    ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
+    JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
+    JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
+    METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
+    MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
+    MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
+    MOCHITEST_WEBAPPRT_CONTENT=('webapprt-content', 'testing/mochitest', 'webapprtContent', True),
+    MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
+    WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
+    XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
+)
+
+# Reftests have their own manifest format and are processed separately.
+REFTEST_FLAVORS = ('crashtest', 'reftest')
+
+# Web platform tests have their own manifest format and are processed separately.
+WEB_PATFORM_TESTS_FLAVORS = ('web-platform-tests',)
+
+def all_test_flavors():
+    return ([v[0] for v in TEST_MANIFESTS.values()] +
+            list(REFTEST_FLAVORS) +
+            list(WEB_PATFORM_TESTS_FLAVORS))