Bug 976733 - part 2 - add TEST_HARNESS_FILES and convert testing/mochitest/ to use it; r=gps

2014-08-15 14:28:26 -04:00 · 2014-08-15 14:28:26 -04:00 · 908de2c6f1
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@ -53,6 +53,7 @@ from ..frontend.data import (
    SharedLibrary,
    SimpleProgram,
    StaticLibrary,
+    TestHarnessFiles,
    TestManifest,
    VariablePassthru,
    XPIDLFile,
@ -409,6 +410,9 @@ class RecursiveMakeBackend(CommonBackend):
        elif isinstance(obj, Exports):
            self._process_exports(obj, obj.exports, backend_file)

+        elif isinstance(obj, TestHarnessFiles):
+            self._process_test_harness_files(obj, backend_file)
+
        elif isinstance(obj, Resources):
            self._process_resources(obj, obj.resources, backend_file)

@ -861,6 +865,22 @@ class RecursiveMakeBackend(CommonBackend):
            if not os.path.exists(source):
                raise Exception('File listed in EXPORTS does not exist: %s' % source)

+    def _process_test_harness_files(self, obj, backend_file):
+        for path, files in obj.srcdir_files.iteritems():
+            for source in files:
+                dest = '%s/%s' % (path, mozpath.basename(source))
+                self._install_manifests['tests'].add_symlink(source, dest)
+
+        for path, files in obj.objdir_files.iteritems():
+            prefix = 'TEST_HARNESS_%s' % path.replace('/', '_')
+            backend_file.write("""
+%(prefix)s_FILES := %(files)s
+%(prefix)s_DEST := %(dest)s
+INSTALL_TARGETS += %(prefix)s
+""" % { 'prefix': prefix,
+        'dest': '$(DEPTH)/_tests/%s' % path,
+        'files': ' '.join(files) })
+
    def _process_resources(self, obj, resources, backend_file):
        dep_path = mozpath.join(self.environment.topobjdir, '_build_manifests', '.deps', 'install')

--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@ -153,6 +153,33 @@ class Context(KeyedDefaultDict):

        return KeyedDefaultDict.__setitem__(self, key, value)

+    def resolve_path(self, path):
+        """Resolves a path using moz.build conventions.
+
+        Paths may be relative to the current srcdir or objdir, or to the
+        environment's topsrcdir or topobjdir.  Different resolution contexts
+        are denoted by characters at the beginning of the path:
+
+            * '/' - relative to topsrcdir;
+            * '!/' - relative to topobjdir;
+            * '!' - relative to objdir; and
+            * any other character - relative to srcdir.
+        """
+        if path.startswith('/'):
+            resolved = mozpath.join(self.config.topsrcdir, path[1:])
+        elif path.startswith('!/'):
+            resolved = mozpath.join(self.config.topobjdir, path[2:])
+        elif path.startswith('!'):
+            resolved = mozpath.join(self.objdir, path[1:])
+        else:
+            resolved = mozpath.join(self.srcdir, path)
+
+        return mozpath.normpath(resolved)
+
+    @staticmethod
+    def is_objdir_path(path):
+        return path[0] == '!'
+
    def update(self, iterable={}, **kwargs):
        """Like dict.update(), but using the context's setitem.

@ -950,6 +977,21 @@ VARIABLES = {

           This variable only has an effect on Windows.
        """, None),
+
+    'TEST_HARNESS_FILES': (HierarchicalStringList, list,
+        """List of files to be installed for test harnesses.
+
+        ``TEST_HARNESS_FILES`` can be used to install files to any directory
+        under $objdir/_tests. Files can be appended to a field to indicate
+        which subdirectory they should be exported to. For example,
+        to export ``foo.py`` to ``_tests/foo``, append to
+        ``TEST_HARNESS_FILES`` like so::
+           TEST_HARNESS_FILES.foo += ['foo.py']
+
+        Files from topsrcdir and the objdir can also be installed by prefixing
+        the path(s) with a '/' character and a '!' character, respectively::
+           TEST_HARNESS_FILES.path += ['/build/bar.py', '!quux.py']
+        """, None),
 }

 # Sanity check: we don't want any variable above to have a list as storage type.
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@ -208,6 +208,21 @@ class Exports(ContextDerived):
        self.exports = exports
        self.dist_install = dist_install

+class TestHarnessFiles(ContextDerived):
+    """Sandbox container object for TEST_HARNESS_FILES,
+    which is a HierarchicalStringList.
+
+    We need an object derived from ContextDerived for use in the backend, so
+    this object fills that role. It just has a reference to the underlying
+    HierarchicalStringList, which is created when parsing TEST_HARNESS_FILES.
+    """
+    __slots__ = ('srcdir_files', 'objdir_files')
+
+    def __init__(self, context, srcdir_files, objdir_files):
+        ContextDerived.__init__(self, context)
+        self.srcdir_files = srcdir_files
+        self.objdir_files = objdir_files
+
 class Resources(ContextDerived):
    """Context derived container object for RESOURCE_FILES, which is a
    HierarchicalStringList, with an extra ``.preprocess`` property on each
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@ -4,6 +4,7 @@

 from __future__ import unicode_literals

+import itertools
 import json
 import logging
 import os
@ -11,7 +12,7 @@ import traceback
 import sys
 import time

-from collections import OrderedDict
+from collections import defaultdict, OrderedDict
 from mach.mixin.logging import LoggingMixin
 from mozbuild.util import (
    memoize,
@ -56,6 +57,7 @@ from .data import (
    SharedLibrary,
    SimpleProgram,
    StaticLibrary,
+    TestHarnessFiles,
    TestWebIDLFile,
    TestManifest,
    VariablePassthru,
@ -486,6 +488,31 @@ class TreeMetadataEmitter(LoggingMixin):
            yield Exports(context, exports,
                dist_install=not context.get('NO_DIST_INSTALL', False))

+        test_harness_files = context.get('TEST_HARNESS_FILES')
+        if test_harness_files:
+            srcdir_files = defaultdict(list)
+            objdir_files = defaultdict(list)
+
+            for path, strings in test_harness_files.walk():
+                if not path and strings:
+                    raise SandboxValidationError(
+                        'Cannot install files to the root of TEST_HARNESS_FILES', context)
+
+                for s in strings:
+                    if context.is_objdir_path(s):
+                        if s.startswith('!/'):
+                            raise SandboxValidationError(
+                                'Topobjdir-relative file not allowed in TEST_HARNESS_FILES: %s' % s, context)
+                        objdir_files[path].append(s[1:])
+                    else:
+                        resolved = context.resolve_path(s)
+                        if not os.path.exists(resolved):
+                            raise SandboxValidationError(
+                                'File listed in TEST_HARNESS_FILES does not exist: %s' % s, context)
+                        srcdir_files[path].append(resolved)
+
+            yield TestHarnessFiles(context, srcdir_files, objdir_files)
+
        defines = context.get('DEFINES')
        if defines:
            yield Defines(context, defines)
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build
@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+TEST_HARNESS_FILES += ["foo.py"]
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini
@ -0,0 +1 @@
+# dummy file so the existence checks for TEST_HARNESS_FILES succeed
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py
@ -0,0 +1 @@
+# dummy file so the existence checks for TEST_HARNESS_FILES succeed
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build
@ -0,0 +1,7 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+TEST_HARNESS_FILES.mochitest += ["runtests.py"]
+TEST_HARNESS_FILES.mochitest += ["utils.py"]
+TEST_HARNESS_FILES.testing.mochitest += ["mochitest.py"]
+TEST_HARNESS_FILES.testing.mochitest += ["mochitest.ini"]
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py
@ -0,0 +1 @@
+# dummy file so the existence checks for TEST_HARNESS_FILES succeed
--- a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py
@ -0,0 +1 @@
+# dummy file so the existence checks for TEST_HARNESS_FILES succeed
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@ -23,12 +23,14 @@ from mozbuild.frontend.data import (
    Resources,
    SimpleProgram,
    StaticLibrary,
+    TestHarnessFiles,
    TestManifest,
    VariablePassthru,
 )
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import (
    BuildReader,
+    BuildReaderError,
    SandboxValidationError,
 )

@ -202,6 +204,29 @@ class TestEmitterBasic(unittest.TestCase):
            self.assertEqual(expect_path, actual_path)
            self.assertEqual(expect_headers, actual_headers)

+    def test_test_harness_files(self):
+        reader = self.reader('test-harness-files')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+        self.assertIsInstance(objs[0], TestHarnessFiles)
+
+        expected = {
+            'mochitest': ['runtests.py', 'utils.py'],
+            'testing/mochitest': ['mochitest.py', 'mochitest.ini'],
+        }
+
+        for path, strings in objs[0].srcdir_files.iteritems():
+            self.assertTrue(path in expected)
+            basenames = sorted(mozpath.basename(s) for s in strings)
+            self.assertEqual(sorted(expected[path]), basenames)
+
+    def test_test_harness_files_root(self):
+        reader = self.reader('test-harness-files-root')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Cannot install files to the root of TEST_HARNESS_FILES'):
+            objs = self.read_topsrcdir(reader)
+
    def test_resources(self):
        reader = self.reader('resources')
        objs = self.read_topsrcdir(reader)
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@ -10,90 +10,7 @@ DIST_FILES = install.rdf
 # Used in install.rdf
 USE_EXTENSION_MANIFEST = 1

-# files that get copied into $objdir/_tests/
-SERV_FILES = 	\
-		runtests.py \
-		bisection.py \
-		automation.py \
-		runtestsb2g.py \
-		runtestsremote.py \
-		mochitest_options.py \
-		manifest.webapp \
-		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanager.py \
-		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py \
-		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py \
-		$(topsrcdir)/testing/mozbase/manifestparser/manifestparser/manifestparser.py \
-		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/droid.py \
-		$(topsrcdir)/testing/mozbase/mozdevice/mozdevice/Zeroconf.py \
-		$(topsrcdir)/testing/mozbase/moznetwork/moznetwork/moznetwork.py \
-		$(topsrcdir)/build/automationutils.py \
-		$(topsrcdir)/build/mobile/remoteautomation.py \
-		gen_template.pl \
-		server.js \
-		chunkifyTests.js \
-		manifestLibrary.js \
-		harness.xul \
-		browser-test-overlay.xul \
-		browser-test.js \
-		cc-analyzer.js \
-		chrome-harness.js \
-		browser-harness.xul \
-		redirect.html \
-		$(topsrcdir)/build/pgo/server-locations.txt \
-		$(topsrcdir)/netwerk/test/httpserver/httpd.js \
-		pywebsocket_wrapper.py \
-		android.json \
-		androidx86.json \
-		android23.json \
-		gl.json \
-		b2g_start_script.js \
-		$(topsrcdir)/build/sanitizers/lsan_suppressions.txt \
-		$(NULL)
-
-ifeq ($(MOZ_BUILD_APP),mobile/android)
-SERV_FILES += \
-  $(topsrcdir)/mobile/android/base/tests/robocop.ini \
-  $(topsrcdir)/mobile/android/base/tests/robocop_autophone.ini \
-  $(NULL)
-endif
-
 _DEST_DIR = $(DEPTH)/_tests/$(relativesrcdir)
-SERV_DEST = $(_DEST_DIR)
-INSTALL_TARGETS += SERV
-
-PYWEBSOCKET_FILES = \
-		pywebsocket/standalone.py \
-		$(NULL)
-PYWEBSOCKET_DEST = $(_DEST_DIR)/pywebsocket
-INSTALL_TARGETS += PYWEBSOCKET
-
-MOD_PYWEBSOCKET_FILES = \
-		pywebsocket/mod_pywebsocket/__init__.py \
-		pywebsocket/mod_pywebsocket/common.py \
-		pywebsocket/mod_pywebsocket/dispatch.py \
-		pywebsocket/mod_pywebsocket/extensions.py \
-		pywebsocket/mod_pywebsocket/headerparserhandler.py \
-		pywebsocket/mod_pywebsocket/http_header_util.py \
-		pywebsocket/mod_pywebsocket/memorizingfile.py \
-		pywebsocket/mod_pywebsocket/util.py \
-		pywebsocket/mod_pywebsocket/stream.py \
-		pywebsocket/mod_pywebsocket/_stream_hixie75.py \
-		pywebsocket/mod_pywebsocket/msgutil.py \
-		pywebsocket/mod_pywebsocket/_stream_hybi.py \
-		pywebsocket/mod_pywebsocket/_stream_base.py \
-		$(NULL)
-MOD_PYWEBSOCKET_DEST = $(_DEST_DIR)/pywebsocket/mod_pywebsocket
-INSTALL_TARGETS += MOD_PYWEBSOCKET
-
-HANDSHAKE_FILES = \
-		pywebsocket/mod_pywebsocket/handshake/__init__.py \
-		pywebsocket/mod_pywebsocket/handshake/hybi00.py \
-		pywebsocket/mod_pywebsocket/handshake/_base.py \
-		pywebsocket/mod_pywebsocket/handshake/draft75.py \
-		pywebsocket/mod_pywebsocket/handshake/hybi.py \
-		$(NULL)
-HANDSHAKE_DEST = $(_DEST_DIR)/pywebsocket/mod_pywebsocket/handshake
-INSTALL_TARGETS += HANDSHAKE

 include $(topsrcdir)/config/rules.mk
 # We're installing to _tests/testing/mochitest, so this is the depth
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@ -25,3 +25,76 @@ MOCHITEST_MANIFESTS += [
    'tests/MochiKit-1.4.2/tests/mochitest.ini',
 ]
 MOCHITEST_CHROME_MANIFESTS += ['chrome/chrome.ini']
+
+TEST_HARNESS_FILES.testing.mochitest += [
+    '!automation.py',
+    '/build/automationutils.py',
+    '/build/mobile/remoteautomation.py',
+    '/build/pgo/server-locations.txt',
+    '/build/sanitizers/lsan_suppressions.txt',
+    '/netwerk/test/httpserver/httpd.js',
+    '/testing/mozbase/manifestparser/manifestparser/manifestparser.py',
+    '/testing/mozbase/mozdevice/mozdevice/devicemanager.py',
+    '/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py',
+    '/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py',
+    '/testing/mozbase/mozdevice/mozdevice/droid.py',
+    '/testing/mozbase/mozdevice/mozdevice/Zeroconf.py',
+    '/testing/mozbase/moznetwork/moznetwork/moznetwork.py',
+    'android.json',
+    'android23.json',
+    'androidx86.json',
+    'b2g_start_script.js',
+    'bisection.py',
+    'browser-harness.xul',
+    'browser-test-overlay.xul',
+    'browser-test.js',
+    'cc-analyzer.js',
+    'chrome-harness.js',
+    'chunkifyTests.js',
+    'gen_template.pl',
+    'gl.json',
+    'harness.xul',
+    'manifest.webapp',
+    'manifestLibrary.js',
+    'mochitest_options.py',
+    'pywebsocket_wrapper.py',
+    'redirect.html',
+    'runtests.py',
+    'runtestsb2g.py',
+    'runtestsremote.py',
+    'server.js',
+]
+
+if CONFIG['MOZ_BUILD_APP'] == 'mobile/android':
+    TEST_HARNESS_FILES.testing.mochitest += [
+        '/mobile/android/base/tests/robocop.ini',
+        '/mobile/android/base/tests/robocop_autophone.ini',
+    ]
+
+TEST_HARNESS_FILES.testing.mochitest.pywebsocket += [
+    'pywebsocket/standalone.py',
+]
+
+TEST_HARNESS_FILES.testing.mochitest.pywebsocket.mod_pywebsocket += [
+    'pywebsocket/mod_pywebsocket/__init__.py',
+    'pywebsocket/mod_pywebsocket/_stream_base.py',
+    'pywebsocket/mod_pywebsocket/_stream_hixie75.py',
+    'pywebsocket/mod_pywebsocket/_stream_hybi.py',
+    'pywebsocket/mod_pywebsocket/common.py',
+    'pywebsocket/mod_pywebsocket/dispatch.py',
+    'pywebsocket/mod_pywebsocket/extensions.py',
+    'pywebsocket/mod_pywebsocket/headerparserhandler.py',
+    'pywebsocket/mod_pywebsocket/http_header_util.py',
+    'pywebsocket/mod_pywebsocket/memorizingfile.py',
+    'pywebsocket/mod_pywebsocket/msgutil.py',
+    'pywebsocket/mod_pywebsocket/stream.py',
+    'pywebsocket/mod_pywebsocket/util.py',
+]
+
+TEST_HARNESS_FILES.testing.mochitest.pywebsocket.mod_pywebsocket.handshake += [
+    'pywebsocket/mod_pywebsocket/handshake/__init__.py',
+    'pywebsocket/mod_pywebsocket/handshake/_base.py',
+    'pywebsocket/mod_pywebsocket/handshake/draft75.py',
+    'pywebsocket/mod_pywebsocket/handshake/hybi.py',
+    'pywebsocket/mod_pywebsocket/handshake/hybi00.py',
+]
				`@ -0,0 +1 @@`
				`# dummy file so the existence checks for TEST_HARNESS_FILES succeed`