From d8e33f0d597e0d50f02f3b1a0175b1305cfd57ab Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 17 Jun 2014 17:29:57 -0700 Subject: [PATCH 01/21] Bug 1027215 - Generic Python code for parsing reftest manifests; r=roc The immediate goal of this patch is to give the build system and testing tools the knowledge to identify reftest files and directories. Parsing extra metadata out of reftest manifests is currently a non-requirement, but may be supported some day. --HG-- extra : rebase_source : 279680af28c9175f5babe458a57203e8b19ab724 extra : histedit_source : c0e463ea02f87a376ef48e2b25136e5f6be4e61a --- build/mach_bootstrap.py | 1 + build/virtualenv_packages.txt | 1 + layout/tools/reftest/reftest/__init__.py | 125 +++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 layout/tools/reftest/reftest/__init__.py diff --git a/build/mach_bootstrap.py b/build/mach_bootstrap.py index d376d56e9a00..3fe3e4da7450 100644 --- a/build/mach_bootstrap.py +++ b/build/mach_bootstrap.py @@ -38,6 +38,7 @@ SEARCH_PATHS = [ 'config', 'dom/bindings', 'dom/bindings/parser', + 'layout/tools/reftest', 'other-licenses/ply', 'xpcom/idl-parser', 'testing', diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt index 08091e1967c8..610189df7073 100644 --- a/build/virtualenv_packages.txt +++ b/build/virtualenv_packages.txt @@ -17,6 +17,7 @@ mozilla.pth:config mozilla.pth:xpcom/typelib/xpt/tools mozilla.pth:dom/bindings mozilla.pth:dom/bindings/parser +mozilla.pth:layout/tools/reftest moztreedocs.pth:tools/docs copy:build/buildconfig.py packages.txt:testing/mozbase/packages.txt diff --git a/layout/tools/reftest/reftest/__init__.py b/layout/tools/reftest/reftest/__init__.py new file mode 100644 index 000000000000..5044b16a174c --- /dev/null +++ b/layout/tools/reftest/reftest/__init__.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import re + +RE_COMMENT = re.compile(r'\s+#') +RE_HTTP = re.compile(r'HTTP\((\.\.(\/\.\.)*)\)') +RE_PROTOCOL = re.compile(r'^\w+:') +FAILURE_TYPES = ( + 'fails', + 'fails-if', + 'needs-focus', + 'random', + 'random-if', + 'silentfail', + 'silentfail-if', + 'skip', + 'skip-if', + 'slow', + 'slow-if', + 'fuzzy', + 'fuzzy-if', + 'require-or', + 'asserts', + 'asserts-if', +) +PREF_ITEMS = ( + 'pref', + 'test-pref', + 'ref-pref', +) + +class ReftestManifest(object): + """Represents a parsed reftest manifest. + + We currently only capture file information because that is the only thing + tools require. + """ + def __init__(self): + self.path = None + self.dirs = set() + self.files = set() + self.manifests = set() + + def load(self, path): + """Parse a reftest manifest file.""" + normalized = os.path.normpath(os.path.abspath(path)) + self.manifests.add(normalized) + if not self.path: + self.path = normalized + + mdir = os.path.dirname(normalized) + self.dirs.add(mdir) + + with open(path, 'r') as fh: + urlprefix = '' + for line in fh: + line = line.decode('utf-8') + + # Entire line is a comment. + if line.startswith('#'): + continue + + # Comments can begin mid line. Strip them. + m = RE_COMMENT.search(line) + if m: + line = line[:m.start()] + + line = line.strip() + if not line: + continue + + items = line.split() + tests = [] + + for i in range(len(items)): + item = items[i] + + if item.startswith(FAILURE_TYPES): + continue + if item.startswith(PREF_ITEMS): + continue + if item == 'HTTP': + continue + + m = RE_HTTP.match(item) + if m: + # Need to package the referenced directory. + self.dirs.add(os.path.normpath(os.path.join( + mdir, m.group(1)))) + continue + + if item == 'url-prefix': + urlprefix = items[i+1] + break + + if item == 'default-preferences': + break + + if item == 'include': + self.load(os.path.join(mdir, items[i+1])) + break + + if item == 'load' or item == 'script': + tests.append(items[i+1]) + break + + if item == '==' or item == '!=': + tests.extend(items[i+1:i+3]) + break + + for f in tests: + # We can't package about: or data: URIs. + # Discarding data isn't correct for a parser. But retaining + # all data isn't currently a requirement. + if RE_PROTOCOL.match(f): + continue + + test = os.path.normpath(os.path.join(mdir, urlprefix + f)) + self.files.add(test) + self.dirs.add(os.path.dirname(test)) From 1fd0e9d9bd6e8fb51ea8b528acdffb6096f16b20 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 17 Jun 2014 17:37:09 -0700 Subject: [PATCH 02/21] Bug 1027215 - Rewrite print-manifest-dirs.py to use new manifest parser; r=roc Now that we've established a slightly better Python reftest manifest parser, switch the existing manifest parsing code in print-manifest-dirs.py to use it. --HG-- extra : rebase_source : 9559eb5aa47d08b9854a27ed012f2a795abc67f7 extra : histedit_source : 7a9a8942f9310c079a5ea4321a7f4f2ad17008d1 --- layout/tools/reftest/print-manifest-dirs.py | 90 +++++---------------- 1 file changed, 20 insertions(+), 70 deletions(-) diff --git a/layout/tools/reftest/print-manifest-dirs.py b/layout/tools/reftest/print-manifest-dirs.py index 76d95f822b40..0f90c255adeb 100644 --- a/layout/tools/reftest/print-manifest-dirs.py +++ b/layout/tools/reftest/print-manifest-dirs.py @@ -3,77 +3,27 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -import sys, os.path, re - -commentRE = re.compile(r"\s+#") -conditionsRE = re.compile(r"^(fails|needs-focus|random|skip|asserts|slow|require-or|silentfail|pref|test-pref|ref-pref|fuzzy)") -httpRE = re.compile(r"HTTP\((\.\.(\/\.\.)*)\)") -protocolRE = re.compile(r"^\w+:") - -def parseManifest(manifest, dirs): - """Parse the reftest manifest |manifest|, adding all directories containing - tests (and the dirs containing the manifests themselves) to the set |dirs|.""" - manifestdir = os.path.dirname(os.path.abspath(manifest)) - dirs.add(manifestdir) - f = file(manifest) - urlprefix = '' - for line in f: - if line[0] == '#': - continue # entire line was a comment - m = commentRE.search(line) - if m: - line = line[:m.start()] - line = line.strip() - if not line: - continue - items = line.split() - while conditionsRE.match(items[0]): - del items[0] - if items[0] == "HTTP": - del items[0] - m = httpRE.match(items[0]) - if m: - # need to package the dir referenced here - d = os.path.normpath(os.path.join(manifestdir, m.group(1))) - dirs.add(d) - del items[0] - - if items[0] == "url-prefix": - urlprefix = items[1] - continue - elif items[0] == "default-preferences": - continue - elif items[0] == "include": - parseManifest(os.path.join(manifestdir, items[1]), dirs) - continue - elif items[0] == "load" or items[0] == "script": - testURLs = [items[1]] - elif items[0] == "==" or items[0] == "!=": - testURLs = items[1:3] - for u in testURLs: - m = protocolRE.match(u) - if m: - # can't very well package about: or data: URIs - continue - d = os.path.dirname(os.path.normpath(os.path.join(manifestdir, urlprefix + u))) - dirs.add(d) - f.close() +import os +import sys +from reftest import ReftestManifest def printTestDirs(topsrcdir, topmanifests): - """Parse |topmanifests| and print a list of directories containing the tests - within (and the manifests including those tests), relative to |topsrcdir|.""" - topsrcdir = os.path.abspath(topsrcdir) - dirs = set() - for manifest in topmanifests: - parseManifest(manifest, dirs) - for dir in sorted(dirs): - d = dir[len(topsrcdir):].replace('\\','/') - if d[0] == '/': - d = d[1:] - print d + """Parse |topmanifests| and print a list of directories containing the tests + within (and the manifests including those tests), relative to |topsrcdir|. + """ + topsrcdir = os.path.abspath(topsrcdir) + dirs = set() + for path in topmanifests: + m = ReftestManifest() + m.load(path) + dirs |= m.dirs + + for d in sorted(dirs): + d = d[len(topsrcdir):].replace('\\', '/').lstrip('/') + print(d) if __name__ == '__main__': - if len(sys.argv) < 3: - print >>sys.stderr, "Usage: %s topsrcdir reftest.list [reftest.list]*" % sys.argv[0] - sys.exit(1) - printTestDirs(sys.argv[1], sys.argv[2:]) + if len(sys.argv) < 3: + print >>sys.stderr, "Usage: %s topsrcdir reftest.list [reftest.list]*" % sys.argv[0] + sys.exit(1) + printTestDirs(sys.argv[1], sys.argv[2:]) From 347044e6723861c12191d8d19fe90c0bcf6fa57d Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 17 Jun 2014 17:59:03 -0700 Subject: [PATCH 03/21] Bug 1027215 - Add reftest manifests to build config; r=mshal reftest and crashtest manifests can now be added to the build configuration via REFTEST_MANIFESTS and CRASHTEST_MANIFESTS, respectively. The master manifest files have been added to layout/moz.build. This patch enables the deprecation of master reftest manifests but stops short of doing it. In the future, we could declare reftest and crashtest manifests in their nearest moz.build file and generate the master manifest (consisting of a bunch of "include" directives) as part of config.status. --HG-- extra : rebase_source : 3503f787b14b24c38daf577a710e67b583476858 extra : histedit_source : 21e55b3d28ee83afb47f3f779251a13c2a90db5f --- layout/moz.build | 3 ++ .../mozbuild/backend/recursivemake.py | 6 +++ python/mozbuild/mozbuild/frontend/emitter.py | 39 ++++++++++++++++++- .../mozbuild/frontend/sandbox_symbols.py | 12 ++++++ .../crashtest.list | 1 + .../test-manifest-keys-extracted/moz.build | 2 + .../test-manifest-keys-extracted/reftest.list | 1 + .../mozbuild/test/frontend/test_emitter.py | 10 ++++- 8 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list diff --git a/layout/moz.build b/layout/moz.build index 8d29cbfd4f83..c7e7a7599a3d 100644 --- a/layout/moz.build +++ b/layout/moz.build @@ -34,3 +34,6 @@ MOCHITEST_MANIFESTS += [ 'reftests/fonts/mochitest.ini', 'reftests/fonts/mplus/mochitest.ini', ] + +REFTEST_MANIFESTS += ['reftests/reftest.list'] +CRASHTEST_MANIFESTS += ['../testing/crashtest/crashtests.list'] diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py index 539f93de7c27..71bdc7a55f7f 100644 --- a/python/mozbuild/mozbuild/backend/recursivemake.py +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -13,6 +13,7 @@ import types from collections import namedtuple import mozwebidlcodegen +from reftest import ReftestManifest import mozbuild.makeutil as mozmakeutil from mozpack.copier import FilePurger @@ -1075,6 +1076,11 @@ class RecursiveMakeBackend(CommonBackend): (obj.install_prefix, set())) m[1].add(obj.manifest_obj_relpath) + if isinstance(obj.manifest, ReftestManifest): + # Mark included files as part of the build backend so changes + # result in re-config. + self.backend_input_files |= obj.manifest.manifests + def _process_local_include(self, local_include, backend_file): if local_include.startswith('/'): path = '$(topsrcdir)' diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py index 6bc8106fdf1d..e775559e0c1a 100644 --- a/python/mozbuild/mozbuild/frontend/emitter.py +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -15,7 +15,7 @@ from mach.mixin.logging import LoggingMixin import mozpack.path as mozpath import manifestparser - +import reftest import mozinfo from .data import ( @@ -419,6 +419,11 @@ class TreeMetadataEmitter(LoggingMixin): for obj in self._process_test_manifest(sandbox, info, path): yield obj + for flavor in ('crashtest', 'reftest'): + for path in sandbox.get('%s_MANIFESTS' % flavor.upper(), []): + for obj in self._process_reftest_manifest(sandbox, flavor, path): + yield obj + jar_manifests = sandbox.get('JAR_MANIFESTS', []) if len(jar_manifests) > 1: raise SandboxValidationError('While JAR_MANIFESTS is a list, ' @@ -601,6 +606,38 @@ class TreeMetadataEmitter(LoggingMixin): 'manifest file %s: %s' % (path, '\n'.join(traceback.format_exception(*sys.exc_info())))) + def _process_reftest_manifest(self, sandbox, flavor, manifest_path): + manifest_path = mozpath.normpath(manifest_path) + manifest_full_path = mozpath.normpath(mozpath.join( + sandbox['SRCDIR'], manifest_path)) + manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, + sandbox['TOPSRCDIR'])) + + manifest = reftest.ReftestManifest() + manifest.load(manifest_full_path) + + # reftest manifests don't come from manifest parser. But they are + # similar enough that we can use the same emitted objects. Note + # that we don't perform any installs for reftests. + obj = TestManifest(sandbox, manifest_full_path, manifest, + flavor=flavor, install_prefix='%s/' % flavor, + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_path))) + + for test in sorted(manifest.files): + obj.tests.append({ + 'path': test, + 'here': mozpath.dirname(test), + 'manifest': manifest_full_path, + 'name': mozpath.basename(test), + 'head': '', + 'tail': '', + 'support-files': '', + 'subsuite': '', + }) + + yield obj + def _emit_directory_traversal_from_sandbox(self, sandbox): o = DirectoryTraversal(sandbox) o.dirs = sandbox.get('DIRS', []) diff --git a/python/mozbuild/mozbuild/frontend/sandbox_symbols.py b/python/mozbuild/mozbuild/frontend/sandbox_symbols.py index b2e7981f7124..b8213ea4c0ea 100644 --- a/python/mozbuild/mozbuild/frontend/sandbox_symbols.py +++ b/python/mozbuild/mozbuild/frontend/sandbox_symbols.py @@ -599,6 +599,12 @@ VARIABLES = { """List of manifest files defining browser chrome tests. """, None), + 'CRASHTEST_MANIFESTS': (StrictOrderingOnAppendList, list, + """List of manifest files defining crashtests. + + These are commonly named crashtests.list. + """, None), + 'METRO_CHROME_MANIFESTS': (StrictOrderingOnAppendList, list, """List of manifest files defining metro browser chrome tests. """, None), @@ -615,6 +621,12 @@ VARIABLES = { """List of manifest files defining webapprt mochitest chrome tests. """, None), + 'REFTEST_MANIFESTS': (StrictOrderingOnAppendList, list, + """List of manifest files defining reftests. + + These are commonly named reftest.list. + """, None), + 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (StrictOrderingOnAppendList, list, """List of manifest files defining WebRTC signalling tests. """, None), diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list new file mode 100644 index 000000000000..b9d7f2685a8b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list @@ -0,0 +1 @@ +== crashtest1.html crashtest1-ref.html diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build index 6de055cb7b5d..0eb18add57df 100644 --- a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build @@ -7,3 +7,5 @@ METRO_CHROME_MANIFESTS += ['metro.ini'] MOCHITEST_MANIFESTS += ['mochitest.ini'] MOCHITEST_CHROME_MANIFESTS += ['chrome.ini'] XPCSHELL_TESTS_MANIFESTS += ['xpcshell.ini'] +REFTEST_MANIFESTS += ['reftest.list'] +CRASHTEST_MANIFESTS += ['crashtest.list'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list new file mode 100644 index 000000000000..3fc25b296602 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list @@ -0,0 +1 @@ +== reftest1.html reftest1-ref.html diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py index 43c27ede2341..35fe462324f0 100644 --- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -379,7 +379,7 @@ class TestEmitterBasic(unittest.TestCase): objs = [o for o in self.read_topsrcdir(reader) if isinstance(o, TestManifest)] - self.assertEqual(len(objs), 6) + self.assertEqual(len(objs), 8) metadata = { 'a11y.ini': { @@ -436,6 +436,14 @@ class TestEmitterBasic(unittest.TestCase): 'tail2': False, }, }, + 'reftest.list': { + 'flavor': 'reftest', + 'installs': {}, + }, + 'crashtest.list': { + 'flavor': 'crashtest', + 'installs': {}, + }, } for o in objs: From 46e8421108f985608e0d8e011c288947a02f28a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Qu=C3=A8ze?= Date: Fri, 20 Jun 2014 15:50:41 -0300 Subject: [PATCH 04/21] Bug 1022856 - Implement translator provider attribution. r=felipe Parts of the patch written by Felipe, with r=florian --- browser/components/preferences/content.js | 6 ++++++ browser/components/preferences/content.xul | 14 ++++++++++---- .../preferences/in-content/content.js | 6 ++++++ .../preferences/in-content/content.xul | 15 +++++++++++---- .../components/translation/Translation.jsm | 6 ++++++ browser/components/translation/jar.mn | 1 + .../microsoft-translator-attribution.png | Bin 0 -> 3422 bytes .../translation/translation-infobar.xml | 18 +++++++++++++++++- .../themes/linux/preferences/preferences.css | 6 ++++++ .../themes/osx/preferences/preferences.css | 5 +++++ .../themes/shared/translation/infobar.inc.css | 18 ++++++++++++++++++ browser/themes/windows/browser.css | 4 ++++ .../windows/preferences/preferences.css | 6 ++++++ 13 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 browser/components/translation/microsoft-translator-attribution.png diff --git a/browser/components/preferences/content.js b/browser/components/preferences/content.js index 6f21d68e3073..c0aee4b0392f 100644 --- a/browser/components/preferences/content.js +++ b/browser/components/preferences/content.js @@ -188,5 +188,11 @@ var gContentPane = { document.documentElement.openWindow("Browser:TranslationExceptions", "chrome://browser/content/preferences/translation.xul", "", null); + }, + + openTranslationProviderAttribution: function () + { + Components.utils.import("resource:///modules/translation/Translation.jsm"); + Translation.openProviderAttribution(); } }; diff --git a/browser/components/preferences/content.xul b/browser/components/preferences/content.xul index d4f9a4eb5396..89ad253a4e80 100644 --- a/browser/components/preferences/content.xul +++ b/browser/components/preferences/content.xul @@ -144,10 +144,16 @@ oncommand="gContentPane.showLanguages();"/>